feat: 诊断(任务书 18 第一块):诊断数学、四种提交、一次运行跑多个实验、账本与材料、flow 诊断各步
消融默认关,关掉时解释阶段一并跳过;选诊断实验用信息价值减成本。 eval/diagnosis/research.json 为第三块配置。vocab 字段说明的出口文字随提示词一起改,见下一次提交的 prompts/ 与同文件改动(本次含在内)。 Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
This commit is contained in:
+17
-4
@@ -12,6 +12,7 @@ export const FIELD_CATEGORIES = ["用户配置", "方法参数"] as const;
|
||||
export type FieldCategory = (typeof FIELD_CATEGORIES)[number];
|
||||
|
||||
const CLOSED = { additionalProperties: false } as const;
|
||||
const PositiveInteger = Type.Integer({ minimum: 1 });
|
||||
/** 文字字段:不能为空,也不能只有空白。 */
|
||||
const TEXT = { pattern: "\\S" };
|
||||
/**
|
||||
@@ -68,7 +69,7 @@ const RoundZero = Type.Union([
|
||||
/** 消融(方法.md 第十一部分"步骤开关"):关,或开并写选格规则。 */
|
||||
const Ablation = Type.Union([
|
||||
Type.Object(
|
||||
{ enabled: method(Type.Literal(false), "消融关:不列消融格;解释照样写文字,但不给每格的三个数") },
|
||||
{ enabled: method(Type.Literal(false), "消融关:不列消融格,也不写解释(解释就是对消融格的预测,方法.md 4.3 ⑫)") },
|
||||
CLOSED,
|
||||
),
|
||||
Type.Object(
|
||||
@@ -83,6 +84,19 @@ const Ablation = Type.Union([
|
||||
),
|
||||
]);
|
||||
|
||||
/** 诊断(方法.md 4.3 诊 1–诊 4、第十一部分"步骤开关"):关,或开并写两个方法参数(6.11)。 */
|
||||
const Diagnosis = Type.Union([
|
||||
Type.Object({ enabled: method(Type.Literal(false), "诊断关:每轮提案之前不做诊断实验") }, CLOSED),
|
||||
Type.Object(
|
||||
{
|
||||
enabled: method(Type.Literal(true), "诊断开"),
|
||||
perResearcher: method(PositiveInteger, "每人每轮诊断实验上限 A;现行值 2(方法.md 6.11)"),
|
||||
maxExperiments: method(PositiveInteger, "一次诊断最多做几个实验 D;现行值 4(方法.md 6.11)"),
|
||||
},
|
||||
CLOSED,
|
||||
),
|
||||
]);
|
||||
|
||||
/**
|
||||
* 两种流程(方法.md 第十一部分)。完整流程写开关、规则和它才用得到的方法参数;
|
||||
* 朴素循环整套照搬,不能写任何开关、规则,也不写它用不到的 λ、u、衰减(它不预测、不消融)。
|
||||
@@ -96,6 +110,7 @@ const FullFlow = Type.Object(
|
||||
Type.Boolean(),
|
||||
"挑刺与回应开不开(方法.md 4.3 ⑤ ⑥);关掉时不挑刺、不回应,协议照样冻结;现行值 true",
|
||||
),
|
||||
diagnosis: method(Diagnosis, "诊断(方法.md 4.3 诊 1–诊 4)"),
|
||||
ablation: method(Ablation, "消融(方法.md 4.3 ⑬–⑰)"),
|
||||
batchRule: method(
|
||||
Type.Enum(BATCH_RULE_NAMES),
|
||||
@@ -121,14 +136,12 @@ const NaiveFlow = Type.Object(
|
||||
{
|
||||
preset: method(
|
||||
Type.Literal("naive"),
|
||||
"朴素循环:方法.md 第十一部分的对照流程整套照搬,不挑刺、不回应、不预测、均匀随机选批、不消融",
|
||||
"朴素循环:方法.md 第十一部分的对照流程整套照搬,不挑刺、不回应、不预测、均匀随机选批、不诊断、不消融",
|
||||
),
|
||||
},
|
||||
CLOSED,
|
||||
);
|
||||
|
||||
const PositiveInteger = Type.Integer({ minimum: 1 });
|
||||
|
||||
export const ResearchConfig = Type.Object(
|
||||
{
|
||||
studyId: user(
|
||||
|
||||
+41
-8
@@ -1,4 +1,11 @@
|
||||
import { BATCH_RULES, CELL_RULES, type BatchRuleName, type CellRuleName } from "../rules/index.ts";
|
||||
import {
|
||||
BATCH_RULES,
|
||||
CELL_RULES,
|
||||
DIAGNOSIS_RULES,
|
||||
type BatchRuleName,
|
||||
type CellRuleName,
|
||||
type DiagnosisRuleName,
|
||||
} from "../rules/index.ts";
|
||||
import type { ResearchConfig } from "./schema.ts";
|
||||
|
||||
// 步骤清单(docs/结构.md 第三部分,方法.md 第十一部分):把配置里的流程、开关、规则翻译成这份研究要走的步骤。
|
||||
@@ -23,6 +30,13 @@ export type StudyStepId = (typeof STUDY_STEPS)[number];
|
||||
export const ROUND_STEPS = [
|
||||
"openRound", // 1. 开轮:冻结本轮名单和提案上限
|
||||
"carryOverPool", // 1. 开轮:处理留池候选
|
||||
"diagnosisProposal", // 诊 1. 诊断解释与诊断实验
|
||||
"diagnosisDedupe", // 诊 2. 判重合并
|
||||
"diagnosisPrediction", // 诊 2. 诊断预测
|
||||
"selectDiagnosis", // 诊 3. 选诊断实验(程序)
|
||||
"diagnosisImplementation", // 诊 4. 实现、核验、修复
|
||||
"diagnosisRun", // 诊 4. 诊断运行
|
||||
"diagnosisSettle", // 诊 4. 诊断结算
|
||||
"proposal", // 2. 提案
|
||||
"dedupe", // 3. 判重合并
|
||||
"firstPrediction", // 4. 第一次预测(看原始提案)
|
||||
@@ -50,11 +64,11 @@ export const ROUND_STEPS = [
|
||||
export type RoundStepId = (typeof ROUND_STEPS)[number];
|
||||
|
||||
/** 决策点用哪条规则:规则名和版本,从 rules 的登记表取。 */
|
||||
export type RuleRef = { name: BatchRuleName | CellRuleName; version: number };
|
||||
export type RuleRef = { name: BatchRuleName | CellRuleName | DiagnosisRuleName; version: number };
|
||||
|
||||
type Status = { status: "execute" } | { status: "skip"; reason: string };
|
||||
|
||||
/** 一步:执行,或按配置跳过(带原因);只有选批、选格两个决策点带规则,其余为 null。 */
|
||||
/** 一步:执行,或按配置跳过(带原因);只有选批、选格、选诊断实验三个决策点带规则,其余为 null。 */
|
||||
export type Step<S extends string> = { id: S; rule: RuleRef | null } & Status;
|
||||
|
||||
export interface StepList {
|
||||
@@ -82,7 +96,7 @@ export function stepList(config: ResearchConfig): StepList {
|
||||
const flow = config.flow;
|
||||
if (flow.preset === "naive") {
|
||||
const naive = skip("朴素循环不做这一步(方法.md 第十一部分)");
|
||||
return assemble("naive", (id) => (NAIVE_STEPS.has(id) ? EXECUTE : naive), batchRule("uniformRandom"), null);
|
||||
return assemble("naive", (id) => (NAIVE_STEPS.has(id) ? EXECUTE : naive), batchRule("uniformRandom"), null, null);
|
||||
}
|
||||
|
||||
// 完整流程:只有开关关掉的步骤跳过。
|
||||
@@ -102,15 +116,27 @@ export function stepList(config: ResearchConfig): StepList {
|
||||
if (!flow.firstPrediction) off(["firstPrediction"], "配置关掉了第一次预测");
|
||||
// 挑刺与回应关掉时,协议照样冻结(freezeProtocol 不跟着关)。
|
||||
if (!flow.critique) off(["critique", "response"], "配置关掉了挑刺与回应");
|
||||
// 消融关掉时解释照样写文字,只是不给每格的三个数(方法.md 第十一部分"步骤开关"),所以 explanation 不跟着关。
|
||||
// 消融关掉时,解释也不做:解释就是对消融格的预测,⑬–⑰ 整段都不做(方法.md 4.3 ⑫)。
|
||||
if (!flow.ablation.enabled) {
|
||||
off(
|
||||
["listCells", "selectCells", "ablationImplementation", "ablationVerification", "ablationRun", "settle"],
|
||||
[
|
||||
"listCells", "explanation", "selectCells", "ablationImplementation",
|
||||
"ablationVerification", "ablationRun", "settle",
|
||||
],
|
||||
"配置关掉了消融",
|
||||
);
|
||||
}
|
||||
if (!flow.diagnosis.enabled) {
|
||||
off(
|
||||
[
|
||||
"diagnosisProposal", "diagnosisDedupe", "diagnosisPrediction", "selectDiagnosis",
|
||||
"diagnosisImplementation", "diagnosisRun", "diagnosisSettle",
|
||||
],
|
||||
"配置关掉了诊断",
|
||||
);
|
||||
}
|
||||
const cells = flow.ablation.enabled ? cellRule(flow.ablation.cellRule) : null;
|
||||
return assemble("full", (id) => skipped[id] ?? EXECUTE, batchRule(flow.batchRule), cells);
|
||||
return assemble("full", (id) => skipped[id] ?? EXECUTE, batchRule(flow.batchRule), cells, flow.diagnosis.enabled ? diagnosisRule() : null);
|
||||
}
|
||||
|
||||
function batchRule(name: BatchRuleName): RuleRef {
|
||||
@@ -121,13 +147,20 @@ function cellRule(name: CellRuleName): RuleRef {
|
||||
return { name, version: CELL_RULES[name].version };
|
||||
}
|
||||
|
||||
/** 选诊断实验只有一条规则:信息价值减成本(方法.md 6.12),配置里不选。 */
|
||||
function diagnosisRule(): RuleRef {
|
||||
return { name: "valueMinusCost", version: DIAGNOSIS_RULES.valueMinusCost.version };
|
||||
}
|
||||
|
||||
function assemble(
|
||||
preset: StepList["preset"],
|
||||
status: (id: StudyStepId | RoundStepId) => Status,
|
||||
batch: RuleRef,
|
||||
cells: RuleRef | null,
|
||||
diagnosis: RuleRef | null,
|
||||
): StepList {
|
||||
const ruleOf = (id: RoundStepId) => (id === "selectBatch" ? batch : id === "selectCells" ? cells : null);
|
||||
const ruleOf = (id: RoundStepId) =>
|
||||
id === "selectBatch" ? batch : id === "selectCells" ? cells : id === "selectDiagnosis" ? diagnosis : null;
|
||||
return {
|
||||
preset,
|
||||
study: STUDY_STEPS.map((id) => ({ id, rule: null, ...status(id) })),
|
||||
|
||||
+56
-6
@@ -1,7 +1,18 @@
|
||||
import { rmSync } from "node:fs";
|
||||
import type { Content } from "../ledger/index.ts";
|
||||
import { formalRun as runDefinition, judge, prepareJudging, prepareRun } from "../tasks/index.ts";
|
||||
import type { HypothesisMetric, JudgedResult, ProcessResult } from "../vocab/index.ts";
|
||||
import {
|
||||
diagnosisRun as diagnosisDefinition,
|
||||
formalRun as runDefinition,
|
||||
judge,
|
||||
judgeDiagnosis,
|
||||
prepareDiagnosisJudging,
|
||||
prepareDiagnosisRun,
|
||||
prepareJudging,
|
||||
prepareRun,
|
||||
type DiagnosisExperiment,
|
||||
type DiagnosisJudged,
|
||||
} from "../tasks/index.ts";
|
||||
import type { HypothesisMetric, JudgedResult, ProcessResult, RunDefinition } from "../vocab/index.ts";
|
||||
import { checkoutVersion } from "../workspace/index.ts";
|
||||
import { containerName, freshDir, judgeContainerName, runPaths, workspaceRepo } from "./paths.ts";
|
||||
import type { Study } from "./study.ts";
|
||||
@@ -9,6 +20,8 @@ import type { Study } from "./study.ts";
|
||||
// 共用动作二:跑一次正式运行(任务书 12 第 2.7 节)。先登记、再启动、再写结果,前后顺序由这个入口保证
|
||||
// (方法.md 第七、九部分):登记占名额,确认启动才扣次,确认没启动的写结果"没启动"、退还名额。
|
||||
// 判分不扣次数,不登记。恢复时接着等一次已经启动的运行,走同一个 finishRun。
|
||||
// 诊断运行(一次运行跑几个实验,用途 diagnosis)走同一个登记、启动、等待的入口,只是准备目录和判分不同:
|
||||
// 运行记录不带判分(judged 为 null),逐个实验的判分在诊断结算前由 judgeDiagnosisRun 从运行目录做。
|
||||
|
||||
export type RunPlan = Omit<Content<"runPlanned">, "definition">;
|
||||
|
||||
@@ -22,11 +35,30 @@ export type RunOutcome = "done" | "stopped" | { notStarted: string };
|
||||
/** 跑一次正式运行:取出 version 的代码,在任务镜像里跑,判分,写结果。用户停下时不等容器,交回 "stopped"。 */
|
||||
export async function formalRun(s: Study, plan: RunPlan, version: string, metrics: HypothesisMetric[]): Promise<RunOutcome> {
|
||||
const definition = runDefinition(s.task, s.frozen.image, String(s.frozen.seed));
|
||||
return launch(s, plan, definition, metrics, (paths) => {
|
||||
checkoutVersion(workspaceRepo(s.dir), version, freshDir(paths.code));
|
||||
prepareRun(s.task, paths.code, freshDir(paths.run), metrics);
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* 跑一次诊断运行:一个容器、一次登记、一次扣次,里面按顺序跑 experiments 里的实验(总时限平分)。
|
||||
* version 是诊断程序,baselineRunDir 是产出当前基线的那次运行的运行目录(原始产物只读地交给每个实验)。
|
||||
*/
|
||||
export async function diagnosisFormalRun(s: Study, plan: RunPlan, version: string, baselineRunDir: string, experiments: DiagnosisExperiment[]): Promise<RunOutcome> {
|
||||
const definition = diagnosisDefinition(s.task, s.frozen.image, String(s.frozen.seed), experiments.map((e) => e.id));
|
||||
return launch(s, plan, definition, null, (paths) => {
|
||||
checkoutVersion(workspaceRepo(s.dir), version, freshDir(paths.code));
|
||||
prepareDiagnosisRun(s.task, paths.code, baselineRunDir, freshDir(paths.run), experiments);
|
||||
});
|
||||
}
|
||||
|
||||
/** 先登记、准备运行目录、启动,确认启动才写"确认启动";metrics 为 null 的是诊断运行(运行记录不带判分)。 */
|
||||
async function launch(s: Study, plan: RunPlan, definition: RunDefinition, metrics: HypothesisMetric[] | null, prepare: (paths: ReturnType<typeof runPaths>) => void): Promise<RunOutcome> {
|
||||
s.ledger.append("runPlanned", { ...plan, definition });
|
||||
const paths = runPaths(s.dir, plan.run);
|
||||
freshDir(paths.base);
|
||||
checkoutVersion(workspaceRepo(s.dir), version, freshDir(paths.code));
|
||||
prepareRun(s.task, paths.code, freshDir(paths.run), metrics);
|
||||
prepare(paths);
|
||||
const started = await s.runner.start(containerName(s.frozen.config.studyId, plan.run), definition, paths.run);
|
||||
if (!started.started) {
|
||||
s.ledger.append("runResult", { run: plan.run, process: { ending: "notStarted", reason: started.reason }, judged: null });
|
||||
@@ -39,13 +71,14 @@ export async function formalRun(s: Study, plan: RunPlan, version: string, metric
|
||||
/**
|
||||
* 等一次已经确认启动的运行结束,判分,写结果,删掉容器。日志写进这次运行自己的日志目录。
|
||||
* 收到"停下"信号就不等了,直接交回 "stopped";容器不管它,下次启动时恢复入口再来接着等。
|
||||
* metrics 为 null 的是诊断运行:这里不判分,结果的 judged 写 null。
|
||||
*/
|
||||
export async function finishRun(s: Study, run: string, metrics: HypothesisMetric[]): Promise<"done" | "stopped"> {
|
||||
export async function finishRun(s: Study, run: string, metrics: HypothesisMetric[] | null): Promise<"done" | "stopped"> {
|
||||
const name = containerName(s.frozen.config.studyId, run);
|
||||
const paths = runPaths(s.dir, run);
|
||||
const waited = await untilStopped(s.stop, s.runner.wait(name, emptyDir(paths.logs)));
|
||||
if (waited === "stopped") return "stopped";
|
||||
const judged = await judgeRun(s, run, waited, metrics);
|
||||
const judged = metrics === null ? null : await judgeRun(s, run, waited, metrics);
|
||||
s.ledger.append("runResult", { run, process: waited, judged });
|
||||
await s.runner.remove(name);
|
||||
return "done";
|
||||
@@ -68,6 +101,23 @@ async function judgeRun(s: Study, run: string, formal: ProcessResult, metrics: H
|
||||
return judge(s.task, paths.run, paths.judge, formal, validator, metrics);
|
||||
}
|
||||
|
||||
/**
|
||||
* 诊断运行的判分(诊断结算前做):每个交出答案的实验单独判分。同 judgeRun:判分目录每次重新建空的,
|
||||
* 判分容器还在就先删掉再用;恢复时从运行目录重新判,不存半截结果。experiments 是这次运行跑的那几个实验,次序同运行。
|
||||
*/
|
||||
export async function judgeDiagnosisRun(s: Study, run: string, process: ProcessResult, experiments: DiagnosisExperiment[]): Promise<DiagnosisJudged> {
|
||||
const paths = runPaths(s.dir, run);
|
||||
const step = prepareDiagnosisJudging(s.task, paths.run, emptyDir(paths.judge), s.frozen.image, process, experiments);
|
||||
if (step.status === "judged") return step.result;
|
||||
const name = judgeContainerName(s.frozen.config.studyId, run);
|
||||
if ((await s.runner.status(name)) !== "absent") await s.runner.remove(name);
|
||||
const started = await s.runner.start(name, step.definition, paths.judge);
|
||||
if (!started.started) throw new Error(`诊断运行 ${run} 的校验器容器没有启动:${started.reason}`);
|
||||
const validator = await s.runner.wait(name, emptyDir(paths.judgeLogs));
|
||||
await s.runner.remove(name);
|
||||
return judgeDiagnosis(s.task, paths.run, paths.judge, process, validator, experiments);
|
||||
}
|
||||
|
||||
/** 清空重建一个目录:恢复时会再等一次、再判一次,上一次没写完的日志和判分目录不留。 */
|
||||
function emptyDir(dir: string): string {
|
||||
rmSync(dir, { recursive: true, force: true });
|
||||
|
||||
+17
-2
@@ -1,7 +1,7 @@
|
||||
import { mkdirSync } from "node:fs";
|
||||
import { resolve } from "node:path";
|
||||
import type { ResearchConfig } from "../config/index.ts";
|
||||
import { forwardNotRunEvent, cellNotRunEvent } from "../materials/index.ts";
|
||||
import { cellNotRunEvent, diagnosisNotRunEvent, forwardNotRunEvent } from "../materials/index.ts";
|
||||
import { Ledger } from "../ledger/index.ts";
|
||||
import { Runner } from "../runner/index.ts";
|
||||
import { fullConfig, readFrozen } from "./frozen.ts";
|
||||
@@ -10,6 +10,7 @@ import { ledgerFile } from "./paths.ts";
|
||||
import { recover } from "./recover.ts";
|
||||
import { baselineRun, saveBaseline } from "./steps/baseline.ts";
|
||||
import { ablationRun, selectCells, settle, unsealChosen } from "./steps/ablation.ts";
|
||||
import { diagnosisDedupe, diagnosisRun, diagnosisSettle, selectDiagnosisExperiments } from "./steps/diagnosis.ts";
|
||||
import { forwardRun, adoptBatch } from "./steps/forward.ts";
|
||||
import { dispatchStage } from "./steps/dispatch.ts";
|
||||
import { dedupe, freezeProtocols, selectBatch } from "./steps/round.ts";
|
||||
@@ -80,10 +81,12 @@ async function doStep(s: Study, next: Exclude<Next, { do: "freezeConfig" }>): Pr
|
||||
return null;
|
||||
case "baselineRun":
|
||||
case "forwardRun":
|
||||
case "ablationRun": {
|
||||
case "ablationRun":
|
||||
case "diagnosisRun": {
|
||||
const ran =
|
||||
next.do === "baselineRun" ? await baselineRun(s, next.run)
|
||||
: next.do === "forwardRun" ? await forwardRun(s, next.round, next.candidate, next.run)
|
||||
: next.do === "diagnosisRun" ? await diagnosisRun(s, next.round, next.run)
|
||||
: await ablationRun(s, next.round, next.cell, next.run);
|
||||
if (ran === "stopped") return USER_STOPPED;
|
||||
if (ran === "done") return null;
|
||||
@@ -103,6 +106,18 @@ async function doStep(s: Study, next: Exclude<Next, { do: "freezeConfig" }>): Pr
|
||||
case "assignEntries":
|
||||
assignEntries(s);
|
||||
return null;
|
||||
case "diagnosisDedupe":
|
||||
await diagnosisDedupe(s, next.round);
|
||||
return null;
|
||||
case "selectDiagnosis":
|
||||
selectDiagnosisExperiments(s, next.round);
|
||||
return null;
|
||||
case "diagnosisNotRun":
|
||||
s.ledger.append("event", diagnosisNotRunEvent(next.round, next.reason));
|
||||
return null;
|
||||
case "diagnosisSettle":
|
||||
await diagnosisSettle(s, next.round);
|
||||
return null;
|
||||
case "dedupe":
|
||||
await dedupe(s, next.round);
|
||||
return null;
|
||||
|
||||
+24
-9
@@ -4,8 +4,9 @@ import { runSession, type SessionTask, type SessionEnding, type SubmitAttempt, t
|
||||
import { Board, buildMaterial, callId, queryMaterial, researcherId, type TaskMaterial } from "../materials/index.ts";
|
||||
import { receive, type Incoming } from "../submissions/index.ts";
|
||||
import { SUBMISSION_KINDS, type JevJudgment, type Json, type SubmissionKind } from "../vocab/index.ts";
|
||||
import { placeBaselineArtifact } from "../tasks/index.ts";
|
||||
import { checkoutVersion } from "../workspace/index.ts";
|
||||
import { prepareSubmission, taskCode, lightChecker } from "./task-code.ts";
|
||||
import { prepareSubmission, taskArtifact, taskCode, lightChecker } from "./task-code.ts";
|
||||
import { callPaths, freshDir, runPaths, workspaceRepo } from "./paths.ts";
|
||||
import type { Study } from "./study.ts";
|
||||
|
||||
@@ -43,7 +44,7 @@ export async function runModelTask<K extends SubmissionKind>(s: Study, t: ModelT
|
||||
prompts: s.prompts,
|
||||
codeRoot: paths.code,
|
||||
changes,
|
||||
artifacts: t.kind === "explanation" ? runArtifacts(s, t.stageId) : null,
|
||||
artifacts: t.kind === "explanation" ? roundArtifacts(s, t.stageId) : t.kind === "diagnosisProposal" ? baselineArtifacts(s, t.stageId) : null,
|
||||
});
|
||||
const model = role === "researcher" ? researcherModel(s, t.actor) : s.frozen.config.models[role];
|
||||
// 先登记、后建目录:调用编号只在登记之后才算用掉,被杀在两者之间时下次的编号不会撞上旧目录。
|
||||
@@ -60,6 +61,8 @@ export async function runModelTask<K extends SubmissionKind>(s: Study, t: ModelT
|
||||
for (const r of material.readable) checkoutVersion(workspaceRepo(s.dir), r.version, freshDir(r.dir));
|
||||
const workDir = freshDir(paths.work);
|
||||
if (startVersion !== null) checkoutVersion(workspaceRepo(s.dir), startVersion, workDir);
|
||||
const artifactFrom = taskArtifact(s, t);
|
||||
if (artifactFrom !== null) placeBaselineArtifact(s.task, artifactFrom, workDir);
|
||||
let reply = 0;
|
||||
const common = {
|
||||
model,
|
||||
@@ -90,22 +93,34 @@ export async function runModelTask<K extends SubmissionKind>(s: Study, t: ModelT
|
||||
* 解释任务要的产物(materials 的 TaskRequest.artifacts):本轮每次已公开正向运行的 metrics.json、solution.json
|
||||
* 和标准输出、标准错误日志(运行编号 → 文件名 → 文字)。完整代码那次的答案出自哪里,就靠这些产物判断。
|
||||
*/
|
||||
function runArtifacts(s: Study, stageId: string): Record<string, Record<string, string>> {
|
||||
function roundArtifacts(s: Study, stageId: string): Record<string, Record<string, string>> {
|
||||
const board = new Board(s.ledger.records());
|
||||
const round = board.stage(stageId).round;
|
||||
const artifacts: Record<string, Record<string, string>> = {};
|
||||
for (const r of board.runs()) {
|
||||
if (r.plan.purpose !== "forward" || r.plan.round !== round || !board.runPublic(r)) continue;
|
||||
const paths = runPaths(s.dir, r.plan.run);
|
||||
const files: Record<string, string> = {};
|
||||
for (const [dir, name] of [[paths.run, "metrics.json"], [paths.run, "solution.json"], [paths.logs, "stdout.log"], [paths.logs, "stderr.log"]] as const) {
|
||||
if (existsSync(join(dir, name))) files[name] = readFileSync(join(dir, name), "utf8");
|
||||
}
|
||||
artifacts[r.plan.run] = files;
|
||||
artifacts[r.plan.run] = runFiles(s, r.plan.run);
|
||||
}
|
||||
return artifacts;
|
||||
}
|
||||
|
||||
/** 诊 1 要的产物:产出本轮当前基线的那次运行,同样的几个文件(运行编号 → 文件名 → 文字)。 */
|
||||
function baselineArtifacts(s: Study, stageId: string): Record<string, Record<string, string>> {
|
||||
const board = new Board(s.ledger.records());
|
||||
const run = board.runOf(board.baselineOf(board.stage(stageId).round))!.plan.run;
|
||||
return { [run]: runFiles(s, run) };
|
||||
}
|
||||
|
||||
/** 一次运行留下的产物:metrics.json、solution.json 和两份日志,有哪个读哪个。 */
|
||||
function runFiles(s: Study, run: string): Record<string, string> {
|
||||
const paths = runPaths(s.dir, run);
|
||||
const files: Record<string, string> = {};
|
||||
for (const [dir, name] of [[paths.run, "metrics.json"], [paths.run, "solution.json"], [paths.logs, "stdout.log"], [paths.logs, "stderr.log"]] as const) {
|
||||
if (existsSync(join(dir, name))) files[name] = readFileSync(join(dir, name), "utf8");
|
||||
}
|
||||
return files;
|
||||
}
|
||||
|
||||
/** 研究者用的模型:第 i 位研究者(researcherId(i))用配置里第 i 个。 */
|
||||
function researcherModel(s: Study, actor: string) {
|
||||
const models = s.frozen.config.models.researchers;
|
||||
|
||||
+111
-7
@@ -1,8 +1,8 @@
|
||||
import type { RoundStepId, StudyStepId } from "../config/index.ts";
|
||||
import type { Content, LedgerRecord } from "../ledger/index.ts";
|
||||
import { Board, CELL_NOT_RUN, cellId, DECISION_KINDS, experimenterId, factorKey, FORWARD_NOT_RUN, ORIGINAL_BASELINE, researcherId, splitCell, verifierId } from "../materials/index.ts";
|
||||
import { Board, CELL_NOT_RUN, cellId, DECISION_KINDS, DIAGNOSIS_NOT_RUN, diagnosisId, experimenterId, factorKey, FORWARD_NOT_RUN, ORIGINAL_BASELINE, researcherId, splitCell, verifierId } from "../materials/index.ts";
|
||||
import { compareIds, listCells, runBalance, sharedSettings } from "../rules/index.ts";
|
||||
import { ablationOutcome, verificationOutcome } from "../submissions/index.ts";
|
||||
import { ablationOutcome, diagnosisOutcome, verificationOutcome } from "../submissions/index.ts";
|
||||
import { SUBMISSION_KINDS, type Stage, type SubmissionKind } from "../vocab/index.ts";
|
||||
import { readFrozen, type Frozen } from "./frozen.ts";
|
||||
|
||||
@@ -23,7 +23,7 @@ const ROUND_ZERO_STAGES = {
|
||||
type RoundZeroStep = keyof typeof ROUND_ZERO_STAGES;
|
||||
|
||||
/** 第 r 轮的研究者阶段:步骤名就是阶段名(vocab STAGES)。 */
|
||||
type RoundStageStep = Extract<RoundStepId, "proposal" | "firstPrediction" | "critique" | "response" | "secondPrediction">;
|
||||
type RoundStageStep = Extract<RoundStepId, "diagnosisProposal" | "diagnosisPrediction" | "proposal" | "firstPrediction" | "critique" | "response" | "secondPrediction">;
|
||||
|
||||
/** 阶段编号由 flow 起:r<轮>-<步骤>,一个步骤一个阶段。第 r 轮按配置跳过的步骤也用这个编号记"按配置跳过"。 */
|
||||
export const stageIdOf = (round: number, step: string): string => `r${round}-${step}`;
|
||||
@@ -52,6 +52,11 @@ export type Next =
|
||||
| { do: "closeStage"; stageId: string }
|
||||
| { do: "dispatch"; stageId: string; tasks: TaskRef[] }
|
||||
| { do: "assignEntries" }
|
||||
| { do: "diagnosisDedupe"; round: number }
|
||||
| { do: "selectDiagnosis"; round: number }
|
||||
| { do: "diagnosisNotRun"; round: number; reason: string }
|
||||
| { do: "diagnosisRun"; round: number; run: string }
|
||||
| { do: "diagnosisSettle"; round: number }
|
||||
| { do: "dedupe"; round: number }
|
||||
| { do: "freezeProtocols"; round: number }
|
||||
| { do: "selectBatch"; round: number }
|
||||
@@ -139,6 +144,28 @@ function roundStepNext(step: RoundStepId, round: number, view: View, frozen: Fro
|
||||
case "carryOverPool":
|
||||
// 不写记录:本轮名单和提案上限就是冻结的配置(方法.md 4.3 ①),留池候选由 materials 从记录算出(Board.carried)。
|
||||
return null;
|
||||
case "diagnosisProposal":
|
||||
case "diagnosisPrediction":
|
||||
// 诊 1、诊 2 → 这个阶段的关闭记录。余额不到 2 次(算到本轮开始前)本轮不诊断:不开阶段、不写记录。
|
||||
return diagnosing(view, round, frozen) ? stageNext(stageIdOf(round, step), step, round, (roster) => roundTasks(step, round, roster, view.board), view, frozen, researchers(frozen)) : null;
|
||||
case "diagnosisDedupe":
|
||||
// 诊 2 判重合并 → 本轮的诊断实验判重合并决定(decision,种类 diagnosisDedup)。
|
||||
if (!diagnosing(view, round, frozen)) return null;
|
||||
return view.board.decision("diagnosisDedup", round) === undefined ? { do: "diagnosisDedupe", round } : null;
|
||||
case "selectDiagnosis":
|
||||
// 诊 3 → 本轮的选诊断实验决定(decision,种类 selectDiagnosis),定了不诊断也写。
|
||||
if (!diagnosing(view, round, frozen)) return null;
|
||||
return view.board.decision("selectDiagnosis", round) === undefined ? { do: "selectDiagnosis", round } : null;
|
||||
case "diagnosisImplementation":
|
||||
// 诊 4 实现与核验 → 最近一次核验里至少有一个实验通过,或修复次数用完而一个都不剩、有"不跑"事件。
|
||||
return diagnosisCodeNext(round, view, frozen);
|
||||
case "diagnosisRun":
|
||||
// 诊 4 运行 → 本轮诊断运行最近一次有结果,而且不是"没启动"。
|
||||
return diagnosisProgramOf(view.board, round, frozen) === null ? null : diagnosisRunNext(round, view);
|
||||
case "diagnosisSettle":
|
||||
// 诊 4 结算 → 本轮的诊断结算记录(diagnosisSettlement,按轮次唯一)。
|
||||
if (diagnosisProgramOf(view.board, round, frozen) === null) return null;
|
||||
return view.board.diagnosisSettlement(round) === undefined ? { do: "diagnosisSettle", round } : null;
|
||||
case "proposal":
|
||||
case "firstPrediction":
|
||||
case "critique":
|
||||
@@ -183,9 +210,6 @@ function roundStepNext(step: RoundStepId, round: number, view: View, frozen: Fro
|
||||
// 解释 → 这一轮解释阶段的关闭记录。
|
||||
const adopted = view.board.decision("adopt", round)?.output.adopted ?? null;
|
||||
if (adopted === null || view.board.protocol(adopted).factors.length < 2) return null;
|
||||
if (frozen.steps.round.some((s) => s.id === "listCells" && s.status === "skip")) {
|
||||
return { do: "halt", reason: "配置关掉了消融、又采纳了多因子改法:不交三个数的解释还没写(cells 只按完整流程全开实现,做步骤开关时再补)", objects: [] };
|
||||
}
|
||||
return stageNext(stageIdOf(round, "explanation"), "explanation", round, (roster) => roster.map((actor) => ({ actor, kind: "explanation" as const, object: null })), view, frozen, researchers(frozen));
|
||||
}
|
||||
case "selectCells":
|
||||
@@ -228,6 +252,7 @@ export function poolOf(board: Board, round: number): string[] {
|
||||
|
||||
/**
|
||||
* 第 r 轮一个研究者阶段的任务(任务书 14 第 2.3 节)。同一个人的几个任务按这里的先后跑。
|
||||
* - 诊断解释与实验、诊断预测:每人一个任务,没有对象;
|
||||
* - 提案:每人一个任务,没有对象;
|
||||
* - 第一次预测:每人 × 本轮每个新候选(含自己的;留池候选不做,方法.md 4.3 ④);
|
||||
* - 挑刺:每人 × 本轮每条不是自己写的原提案(监督者定,照参考项目);
|
||||
@@ -238,6 +263,9 @@ function roundTasks(step: RoundStageStep, round: number, roster: readonly string
|
||||
const each = (kind: TaskRef["kind"], objects: readonly string[]): TaskRef[] =>
|
||||
roster.flatMap((actor) => objects.map((object) => ({ actor, kind, object })));
|
||||
switch (step) {
|
||||
case "diagnosisProposal":
|
||||
case "diagnosisPrediction":
|
||||
return roster.map((actor) => ({ actor, kind: step, object: null }));
|
||||
case "proposal":
|
||||
return roster.map((actor) => ({ actor, kind: "proposal", object: null }));
|
||||
case "firstPrediction":
|
||||
@@ -318,7 +346,7 @@ const researchers = (frozen: Frozen): string[] => frozen.config.models.researche
|
||||
function assignedStage(
|
||||
step: string,
|
||||
round: number,
|
||||
kind: "implementation" | "repair" | "verification" | "ablationImplementation" | "ablationVerification",
|
||||
kind: "implementation" | "repair" | "verification" | "ablationImplementation" | "ablationVerification" | "diagnosisImplementation" | "diagnosisRepair" | "diagnosisVerification",
|
||||
candidates: readonly string[],
|
||||
view: View,
|
||||
frozen: Frozen,
|
||||
@@ -355,6 +383,82 @@ function verificationNext(round: number, view: View, frozen: Frozen): Next | nul
|
||||
return null;
|
||||
}
|
||||
|
||||
/**
|
||||
* 第 round 轮诊不诊断的前提(方法.md 4.3 诊断一段):余额至少 2 次,保证本轮至少还能跑 1 次正向。
|
||||
* 余额只算本轮开始前的运行:本轮诊断运行自己扣了次数以后,诊断这一段的后几步不会反过来算成"不诊断"。
|
||||
* 诊断关着的步骤清单把这几步都标了跳过,不会走到这里。
|
||||
*/
|
||||
function diagnosing(view: View, round: number, frozen: Frozen): boolean {
|
||||
return runBalance(frozen.config.runs, view.board.runStatesBefore(round)).balance >= 2;
|
||||
}
|
||||
|
||||
/**
|
||||
* 诊 4 实现、核验与修复(方法.md 4.3 诊 4):一位实验员把选中的全部实验写进一份诊断程序,一位核验员逐个实验核对。
|
||||
* 阶段编号按"第几次"排:诊断实现-1、核验-1,不通过的退回修复-1、再核验-2……第几次从已开关的阶段数出来,不另存;
|
||||
* 每次核验都覆盖全部选中的实验。最近一次核验里还有实验不通过、修复次数又用完时,不通过的实验拿掉(diagnosisProgramOf);
|
||||
* 一个都不剩才写"不跑"事件(方法.md 4.3 诊 4)。
|
||||
*/
|
||||
function diagnosisCodeNext(round: number, view: View, frozen: Frozen): Next | null {
|
||||
if (chosenExperiments(view.board, round) === null) return null;
|
||||
const object = diagnosisId(round);
|
||||
for (let n = 1; ; n++) {
|
||||
const work = n === 1
|
||||
? assignedStage("diagnosisImplementation-1", round, "diagnosisImplementation", [object], view, frozen)
|
||||
: assignedStage(`diagnosisRepair-${n - 1}`, round, "diagnosisRepair", [object], view, frozen);
|
||||
if (work !== null) return work;
|
||||
const verify = assignedStage(`diagnosisVerification-${n}`, round, "diagnosisVerification", [object], view, frozen);
|
||||
if (verify !== null) return verify;
|
||||
const verification = view.board.published("diagnosisVerification").filter((v) => v.round === round).at(-1)!;
|
||||
if (diagnosisOutcome(verification.content).passed) return null;
|
||||
if (n > frozen.config.maxRepairs) {
|
||||
const { experiments, dropped } = diagnosisProgram(view.board, round, frozen.config.maxRepairs);
|
||||
if (experiments.length > 0 || view.events(DIAGNOSIS_NOT_RUN).some((e) => e.objects.includes(object))) return null;
|
||||
return { do: "diagnosisNotRun", round, reason: `选中的实验第 ${n} 次核验都不通过,修复次数上限 ${frozen.config.maxRepairs} 已用完:${dropped.map((d) => `${d.experiment}:${d.reason}`).join(";")}` };
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/** 本轮诊断选中的实验(诊 3 的决定,按实验编号排);本轮没有诊断(余额不到 2 次、定了不诊断)为 null。 */
|
||||
function chosenExperiments(board: Board, round: number): string[] | null {
|
||||
const decision = board.decision("selectDiagnosis", round);
|
||||
return decision === undefined || !decision.output.diagnose ? null : decision.output.chosen;
|
||||
}
|
||||
|
||||
/**
|
||||
* 诊 4 核验做完之后本轮诊断实际要跑的实验:选中的实验里最近一次核验通过的;不通过的实验拿掉,各带原因
|
||||
* (记"证据不足:核验不通过")。拿掉实验后诊断程序的代码不动:核验员核过的就是这一份,只是运行时不调不通过的实验的编号。
|
||||
* 只在诊断实现与核验这一步做完以后调用。
|
||||
*/
|
||||
export function diagnosisProgram(board: Board, round: number, maxRepairs: number): { experiments: string[]; dropped: Array<{ experiment: string; reason: string }> } {
|
||||
const verifications = board.published("diagnosisVerification").filter((s) => s.round === round);
|
||||
const last = verifications.at(-1)!.content;
|
||||
const { failing } = diagnosisOutcome(last);
|
||||
return {
|
||||
experiments: board.decision("selectDiagnosis", round)!.output.chosen.filter((e) => !failing.includes(e)),
|
||||
dropped: failing.map((experiment) => {
|
||||
const x = last.experiments.find((y) => y.experiment === experiment)!;
|
||||
const what = `${x.followed ? "" : "没照写法做;"}${x.nothingElse ? "" : "动了写法以外的地方;"}${x.where}`;
|
||||
return { experiment, reason: `第 ${verifications.length} 次核验仍不通过,修复次数上限 ${maxRepairs} 已用完:${what}` };
|
||||
}),
|
||||
};
|
||||
}
|
||||
|
||||
/** 本轮要跑的诊断实验:本轮没有诊断、或一个都没剩下为 null。 */
|
||||
export function diagnosisProgramOf(board: Board, round: number, frozen: Frozen): string[] | null {
|
||||
if (chosenExperiments(board, round) === null) return null;
|
||||
const { experiments } = diagnosisProgram(board, round, frozen.config.maxRepairs);
|
||||
return experiments.length === 0 ? null : experiments;
|
||||
}
|
||||
|
||||
/** 诊 4 运行:和正向同一套,没启动的下次启动起新编号,已登记没结果的必须先由恢复入口核对。 */
|
||||
function diagnosisRunNext(round: number, view: View): Next | null {
|
||||
const runs = view.board.runs().filter((r) => r.plan.purpose === "diagnosis" && r.plan.round === round);
|
||||
const last = runs.at(-1);
|
||||
if (last !== undefined && last.result === null) throw new Error(`诊断运行 ${last.plan.run} 登记了还没有结果:恢复入口应当已经核对过`);
|
||||
if (last !== undefined && last.result!.process.ending !== "notStarted") return null;
|
||||
return { do: "diagnosisRun", round, run: `r${round}-diagnosis-${runs.length + 1}` };
|
||||
}
|
||||
|
||||
/** ⑩:按选批顺序一次只交回一个运行;没启动的下次启动起新编号,不会把半批提前公开。 */
|
||||
function forwardNext(round: number, view: View): Next | null {
|
||||
for (const candidate of view.board.decision("batch", round)!.output.chosen) {
|
||||
|
||||
+6
-2
@@ -43,9 +43,13 @@ export async function recover(s: Study): Promise<{ halt: string } | "stopped" |
|
||||
return null;
|
||||
}
|
||||
|
||||
/** 恢复和第一次运行共用原登记指标定义(引用指标也保留原来的缺失规则);消融格缺失一律只记缺失(方法.md 第五部分"指标")。 */
|
||||
function metricsOf(s: Study, plan: Content<"runPlanned">): HypothesisMetric[] {
|
||||
/**
|
||||
* 恢复和第一次运行共用原登记指标定义(引用指标也保留原来的缺失规则);消融格缺失一律只记缺失(方法.md 第五部分"指标")。
|
||||
* 诊断运行为 null:运行记录不带判分,逐个实验的判分在诊断结算前做。
|
||||
*/
|
||||
function metricsOf(s: Study, plan: Content<"runPlanned">): HypothesisMetric[] | null {
|
||||
if (plan.purpose === "baseline") return [];
|
||||
if (plan.purpose === "diagnosis") return null;
|
||||
const board = new Board(s.ledger.records());
|
||||
if (plan.purpose === "forward") return protocolMetrics(board, board.protocol(plan.object!));
|
||||
return ablationMetrics(board, splitCell(plan.object!).candidate);
|
||||
|
||||
@@ -0,0 +1,165 @@
|
||||
import { judgeSameExperiment, type DedupExperiment } from "../../jev/index.ts";
|
||||
import { Board, diagnosisId, diagnosisKey, type DiagnosisDedupOutput } from "../../materials/index.ts";
|
||||
import {
|
||||
compareIds,
|
||||
runBalance,
|
||||
runDiagnosisRule,
|
||||
settleDiagnosis,
|
||||
type DiagnosisExplanation,
|
||||
type DiagnosisRuleInput,
|
||||
type DiagnosisRuleName,
|
||||
type ExperimentOutcome,
|
||||
type SettleDiagnosisInput,
|
||||
} from "../../rules/index.ts";
|
||||
import type { DiagnosisExperiment, DiagnosisJudged } from "../../tasks/index.ts";
|
||||
import type { Json } from "../../vocab/index.ts";
|
||||
import { diagnosisFormalRun, judgeDiagnosisRun, type RunOutcome } from "../formal-run.ts";
|
||||
import { diagnosisProgram, diagnosisProgramOf } from "../next-step.ts";
|
||||
import { baselineRunDir } from "../task-code.ts";
|
||||
import type { Study } from "../study.ts";
|
||||
import type { SamePair } from "./round.ts";
|
||||
|
||||
// 诊断这一段里程序自己做的四步(方法.md 4.3 诊 2–诊 4):判重合并、选诊断实验、诊断运行、诊断结算。
|
||||
// 判重合并和选实验各写一条决定,结算写一条诊断结算,写下之前被打断就从账本重新算一遍;写下之后恢复时照原样用,不重算。
|
||||
|
||||
/** 一个作者交的一个诊断实验:标签 "<作者>.e<第几个>",只在判重合并的输入里认实验用。 */
|
||||
type Item = { author: string; index: number; label: string; experiment: DedupExperiment };
|
||||
|
||||
/**
|
||||
* 诊 2 判重合并(方法.md 4.3):本轮已公开的诊断实验两两问一次 Jev,按作者编号、再按第几个排好顺序发,
|
||||
* 全部问完才写决定:中途被打断就整轮重问,判重不扣运行次数。
|
||||
*/
|
||||
export async function diagnosisDedupe(s: Study, round: number): Promise<void> {
|
||||
const board = new Board(s.ledger.records());
|
||||
const model = s.frozen.config.models.jev;
|
||||
const items: Item[] = board
|
||||
.published("diagnosisProposal")
|
||||
.filter((p) => p.round === round)
|
||||
.sort((a, b) => compareIds(a.author, b.author))
|
||||
.flatMap((p) => p.content.experiments.map((experiment, index) => ({ author: p.author, index, label: `${p.author}.e${index + 1}`, experiment })));
|
||||
const pairs: SamePair[] = [];
|
||||
for (const [i, a] of items.entries()) {
|
||||
for (const b of items.slice(i + 1)) pairs.push({ a: a.label, b: b.label, judgment: await judgeSameExperiment(model, a.experiment, b.experiment) });
|
||||
}
|
||||
// SAFETY: JevJudgment 和 DiagnosisDedupOutput 都是纯 Json 值;账本在写入时校验 Json。
|
||||
s.ledger.append("decision", {
|
||||
decision: "diagnosisDedup",
|
||||
round,
|
||||
rule: "jevSameExperiment",
|
||||
ruleVersion: 1,
|
||||
params: { jevModel: model },
|
||||
seed: null,
|
||||
input: { pairs } as unknown as Json,
|
||||
output: groupSameExperiment(items, pairs) as unknown as Json,
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* 判为相同的连成一组(同判重合并:A 和 B 相同、B 和 C 相同,A、B、C 就是一个实验)。每组合成一个实验,
|
||||
* 按组里排最前的那一个排序,编号用 materials 的 diagnosisKey(d1…);组内按作者编号、再按第几个排,第一个的正文就是合并后的正文。
|
||||
* items 要先按作者编号、第几个排好。
|
||||
*/
|
||||
export function groupSameExperiment(items: readonly Pick<Item, "author" | "index" | "label">[], pairs: readonly SamePair[]): DiagnosisDedupOutput {
|
||||
const parent = new Map(items.map((x) => [x.label, x.label]));
|
||||
const root = (x: string): string => {
|
||||
while (parent.get(x) !== x) x = parent.get(x)!;
|
||||
return x;
|
||||
};
|
||||
for (const p of pairs) if (p.judgment.yes) parent.set(root(p.a), root(p.b));
|
||||
const groups = new Map<string, Array<{ author: string; index: number }>>();
|
||||
for (const x of items) groups.set(root(x.label), [...(groups.get(root(x.label)) ?? []), { author: x.author, index: x.index }]);
|
||||
return { experiments: [...groups.values()].map((sources, i) => ({ experiment: diagnosisKey(i), sources })) };
|
||||
}
|
||||
|
||||
/**
|
||||
* 诊 3 选诊断实验(方法.md 6.12):基线、方向、λ、合并后的全部实验、每条解释对每个实验的三个数,加余额(本轮开始前)、
|
||||
* 每批上限、一次诊断最多几个实验 D、u、衰减,交给步骤清单里 selectDiagnosis 那一步的规则,写一条决定。
|
||||
* 规则没有随机,种子写 null;定了不诊断也写。
|
||||
*/
|
||||
export function selectDiagnosisExperiments(s: Study, round: number): void {
|
||||
const board = new Board(s.ledger.records());
|
||||
const { config, facts, steps } = s.frozen;
|
||||
const { diagnosis } = config.flow;
|
||||
if (!diagnosis.enabled) throw new Error("配置关掉了诊断,步骤清单里选诊断实验一步应当标了跳过");
|
||||
const input: DiagnosisRuleInput = {
|
||||
baseline: board.baselineScore(board.baselineOf(round)),
|
||||
direction: facts.finalMetric.direction,
|
||||
lambda: config.flow.openReserve,
|
||||
experiments: board.diagnosisExperiments(round).map((e) => e.id),
|
||||
explanations: predictionsOf(board, round),
|
||||
balance: runBalance(config.runs, board.runStatesBefore(round)).balance,
|
||||
batchCap: config.batchLimit,
|
||||
maxExperiments: diagnosis.maxExperiments,
|
||||
u: config.flow.knowledgeUse,
|
||||
decay: config.flow.decay,
|
||||
};
|
||||
const rule = steps.round.find((x) => x.id === "selectDiagnosis")!.rule!.name as DiagnosisRuleName;
|
||||
const { rule: used, ...output } = runDiagnosisRule(rule, input);
|
||||
// SAFETY: rules 是纯计算,输入输出都是纯 Json 值;账本在写入时校验 Json。
|
||||
s.ledger.append("decision", {
|
||||
decision: "selectDiagnosis",
|
||||
round,
|
||||
rule: used.name,
|
||||
ruleVersion: used.version,
|
||||
params: output.params,
|
||||
seed: null,
|
||||
input: input as unknown as Json,
|
||||
output: output as unknown as Json,
|
||||
});
|
||||
}
|
||||
|
||||
/** 这一轮已冻结的各条诊断解释:作者的第 i 条(materials 的 diagnosisExplanationId),每条对每个合并后的实验给的三个数。选实验和结算共用。 */
|
||||
function predictionsOf(board: Board, round: number): DiagnosisExplanation[] {
|
||||
const predictions = board.published("diagnosisPrediction").filter((p) => p.round === round);
|
||||
return board.diagnosisExplanations(round).map((x) => ({
|
||||
id: x.id,
|
||||
author: x.author,
|
||||
experiments: predictions.find((p) => p.author === x.author)!.content.predictions.filter((y) => y.explanation === x.id).map((y) => ({ experiment: y.experiment, range: y.range })),
|
||||
}));
|
||||
}
|
||||
|
||||
/** 要跑的实验的运行输入:编号加它登记的假设专用指标(全名),次序同 ids,也就是运行的次序。 */
|
||||
function experimentsOf(board: Board, round: number, ids: readonly string[]): DiagnosisExperiment[] {
|
||||
const merged = board.diagnosisExperiments(round);
|
||||
return ids.map((id) => ({ id, metrics: merged.find((e) => e.id === id)!.metrics }));
|
||||
}
|
||||
|
||||
/**
|
||||
* 诊 4 诊断运行:用核验过的诊断程序,复用正式运行的登记、扣次入口,对象是本轮诊断。跑的只是核验里通过的实验:
|
||||
* 不通过而拿掉的实验,代码留在程序里不动(核验员核过的就是这一份),运行时不调它的编号;每个实验分到总时限 ÷ 跑的实验数。
|
||||
*/
|
||||
export function diagnosisRun(s: Study, round: number, run: string): Promise<RunOutcome> {
|
||||
const board = new Board(s.ledger.records());
|
||||
const ran = diagnosisProgramOf(board, round, s.frozen)!;
|
||||
const version = board.code(diagnosisId(round)).version;
|
||||
return diagnosisFormalRun(s, { run, purpose: "diagnosis", round, object: diagnosisId(round) }, version, baselineRunDir(s, board, round), experimentsOf(board, round, ran));
|
||||
}
|
||||
|
||||
/**
|
||||
* 诊 4 结算(方法.md 4.3 诊 4、6.12):先从运行目录给每个交出答案的实验单独判分,再照 SettleDiagnosisInput 拼输入,
|
||||
* 写一条诊断结算。每个合并后的实验的结局:跑了且有效分数是有效;跑了没有效分数、或核验不通过拿掉的是证据不足;没选中的是没做。
|
||||
* 运行记录不带判分,所以被打断在判分和写结算之间时,恢复后从运行目录重新判。
|
||||
*/
|
||||
export async function diagnosisSettle(s: Study, round: number): Promise<void> {
|
||||
const board = new Board(s.ledger.records());
|
||||
const { experiments: ran, dropped } = diagnosisProgram(board, round, s.frozen.config.maxRepairs);
|
||||
const run = board.runs().filter((r) => r.plan.purpose === "diagnosis" && r.plan.round === round).at(-1)!;
|
||||
const judged = await judgeDiagnosisRun(s, run.plan.run, run.result!.process, experimentsOf(board, round, ran));
|
||||
const input: SettleDiagnosisInput = {
|
||||
baseline: board.baselineScore(board.baselineOf(round)),
|
||||
direction: s.frozen.facts.finalMetric.direction,
|
||||
epsilon: s.frozen.facts.epsilon,
|
||||
lambda: s.frozen.config.flow.openReserve,
|
||||
experiments: board.diagnosisExperiments(round).map((e) => ({ experiment: e.id, outcome: outcomeOf(e.id, ran, dropped.map((d) => d.experiment), judged) })),
|
||||
explanations: predictionsOf(board, round),
|
||||
};
|
||||
// SAFETY: 判分结果、结算的输入输出都是纯 Json 值;账本在写入时校验 Json。
|
||||
s.ledger.append("diagnosisSettlement", { round, result: { run: run.plan.run, judged, dropped, input, output: settleDiagnosis(input) } as unknown as Json });
|
||||
}
|
||||
|
||||
function outcomeOf(experiment: string, ran: readonly string[], dropped: readonly string[], judged: DiagnosisJudged): ExperimentOutcome {
|
||||
if (dropped.includes(experiment)) return { kind: "insufficient" };
|
||||
if (!ran.includes(experiment)) return { kind: "untested" };
|
||||
const j = judged.experiments.find((x) => x.id === experiment)!;
|
||||
return j.verdict === "valid" && j.score.status === "measured" ? { kind: "valid", score: j.score.value } : { kind: "insufficient" };
|
||||
}
|
||||
@@ -94,6 +94,7 @@ function studyFacts(frozen: Frozen, task: Task): StudyFacts {
|
||||
lambda: config.flow.openReserve,
|
||||
totalRuns: config.runs,
|
||||
proposalLimit: config.proposalLimit,
|
||||
diagnosisExperimentLimit: config.flow.diagnosis.enabled ? config.flow.diagnosis.perResearcher : null,
|
||||
factorLimit: config.factorLimit,
|
||||
metrics: metricDefinitions(task),
|
||||
taskText: task.guide,
|
||||
|
||||
+41
-3
@@ -1,19 +1,20 @@
|
||||
import { join } from "node:path";
|
||||
import type { Content } from "../ledger/index.ts";
|
||||
import { codeHunks } from "../jev/index.ts";
|
||||
import { Board, protocolMetrics, splitCell, type CodeChanges } from "../materials/index.ts";
|
||||
import { Board, diagnosisId, protocolMetrics, splitCell, type CodeChanges } from "../materials/index.ts";
|
||||
import type { KnownOf } from "../submissions/index.ts";
|
||||
import { lightCheck, protectedChanges, readLightCheck } from "../tasks/index.ts";
|
||||
import { diagnosisUnversionedNames, lightCheck, protectedChanges, readLightCheck } from "../tasks/index.ts";
|
||||
import type { Json, Submission, SubmissionKind } from "../vocab/index.ts";
|
||||
import { compareVersions, saveVersion } from "../workspace/index.ts";
|
||||
import type { TaskRef } from "./next-step.ts";
|
||||
import { callPaths, containerName, freshDir, workspaceRepo } from "./paths.ts";
|
||||
import { callPaths, containerName, freshDir, runPaths, workspaceRepo } from "./paths.ts";
|
||||
import type { Study } from "./study.ts";
|
||||
|
||||
/**
|
||||
* 实现取本批基线;修复取上次提交。核验、修复的材料差异始终相对本批基线。
|
||||
* 消融实现第一次取被采纳的完整代码,重做取上一次交的这一格代码(materials 的消融实现材料就是这样写的);
|
||||
* 消融核验的材料差异是完整代码到这一格代码(方法.md 4.3 ⑯)。
|
||||
* 诊断实现取当前基线;诊断修复取上次提交的诊断程序;诊断核验、修复的材料差异是当前基线到诊断程序(方法.md 4.3 诊 4)。
|
||||
*/
|
||||
export function taskCode(s: Study, t: TaskRef & { stageId: string }): { changes: CodeChanges | null; startVersion: string | null } {
|
||||
const board = new Board(s.ledger.records());
|
||||
@@ -26,6 +27,13 @@ export function taskCode(s: Study, t: TaskRef & { stageId: string }): { changes:
|
||||
const { candidate } = splitCell(t.object!);
|
||||
return { changes: compareVersions(workspaceRepo(s.dir), board.code(candidate).version, board.code(t.object!).version), startVersion: null };
|
||||
}
|
||||
if (t.kind === "diagnosisImplementation" || t.kind === "diagnosisRepair" || t.kind === "diagnosisVerification") {
|
||||
const round = board.stage(t.stageId).round;
|
||||
const baseline = board.code(board.baselineOf(round)).version;
|
||||
if (t.kind === "diagnosisImplementation") return { changes: null, startVersion: baseline };
|
||||
const program = board.code(t.object!).version;
|
||||
return { changes: compareVersions(workspaceRepo(s.dir), baseline, program), startVersion: t.kind === "diagnosisRepair" ? program : null };
|
||||
}
|
||||
if (t.kind !== "implementation" && t.kind !== "repair" && t.kind !== "verification") return { changes: null, startVersion: null };
|
||||
const baseline = board.code(board.baselineOf(board.stage(t.stageId).round)).version;
|
||||
if (t.kind === "implementation") return { changes: null, startVersion: baseline };
|
||||
@@ -33,6 +41,18 @@ export function taskCode(s: Study, t: TaskRef & { stageId: string }): { changes:
|
||||
return { changes: compareVersions(workspaceRepo(s.dir), baseline, version), startVersion: t.kind === "repair" ? version : null };
|
||||
}
|
||||
|
||||
/** 产出第 round 轮当前基线的那次运行的运行目录:诊断实验要接着算的原始产物就在里面。 */
|
||||
export function baselineRunDir(s: Study, board: Board, round: number): string {
|
||||
return runPaths(s.dir, board.runOf(board.baselineOf(round))!.plan.run).run;
|
||||
}
|
||||
|
||||
/** 诊断实现、修复的工作目录要放进的原始产物所在的运行目录(取出代码之后,tasks 的 placeBaselineArtifact 放);别的任务为 null。 */
|
||||
export function taskArtifact(s: Study, t: TaskRef & { stageId: string }): string | null {
|
||||
if (t.kind !== "diagnosisImplementation" && t.kind !== "diagnosisRepair") return null;
|
||||
const board = new Board(s.ledger.records());
|
||||
return baselineRunDir(s, board, board.stage(t.stageId).round);
|
||||
}
|
||||
|
||||
/**
|
||||
* 提交时才知道的东西:提案份数、回应原提案,或实验员当前工作目录的快照和相对本批基线的改动。
|
||||
* 快照写入 workspace 后才问 Jev;收下才把 code 和 submission 在同一个事务写进账本。
|
||||
@@ -61,6 +81,24 @@ export function prepareSubmission(
|
||||
code: { object: t.object!, version: saved.version, locations: null },
|
||||
};
|
||||
}
|
||||
if (t.kind === "diagnosisImplementation" || t.kind === "diagnosisRepair") {
|
||||
// 诊断程序:存的名单多跳过宿主放进来的原始产物目录;Jev 对照的是选中实验的原文和要采集的指标。
|
||||
const saved = saveVersion(workspaceRepo(s.dir), workDir, diagnosisUnversionedNames(s.task));
|
||||
if (!saved.saved) return { known: { code: saved }, code: null };
|
||||
const round = board.stage(t.stageId).round;
|
||||
const changes = compareVersions(workspaceRepo(s.dir), board.code(board.baselineOf(round)).version, saved.version);
|
||||
const chosen = board.diagnosisExperiments(round).filter((e) => board.decision("selectDiagnosis", round)!.output.chosen.includes(e.id));
|
||||
const impl = content as Submission<"diagnosisImplementation">;
|
||||
return {
|
||||
known: { code: {
|
||||
saved: true,
|
||||
protectedFiles: protectedChanges(s.task, changes.files.map((f) => f.path)),
|
||||
hunks: codeHunks(changes),
|
||||
protocol: { body: chosen.map((e) => `### ${e.id} ${e.name}\n\n动哪一处:${e.spot}\n\n${e.body}`).join("\n\n"), metrics: chosen.flatMap((e) => e.metrics) },
|
||||
} },
|
||||
code: { object: t.object!, version: saved.version, locations: impl.locations as Json },
|
||||
};
|
||||
}
|
||||
if (t.kind !== "implementation" && t.kind !== "repair") return { known: null, code: null };
|
||||
const saved = saveVersion(workspaceRepo(s.dir), workDir, s.frozen.facts.unversionedNames);
|
||||
if (!saved.saved) return { known: { code: saved }, code: null };
|
||||
|
||||
@@ -1,11 +1,15 @@
|
||||
// jev:问 Jev 的问法和阈值(docs/结构.md 第二部分)。判断的类型 JevJudgment 在 vocab。
|
||||
export {
|
||||
checkExperiment,
|
||||
checkHunk,
|
||||
checkProtocol,
|
||||
codeHunks,
|
||||
judgeSameChange,
|
||||
judgeSameExperiment,
|
||||
type CodeHunk,
|
||||
type DedupExperiment,
|
||||
type DedupFactor,
|
||||
type DiagnosisExperiment,
|
||||
type FrozenProtocol,
|
||||
type JevCheck,
|
||||
type ProtocolMetric,
|
||||
|
||||
+37
-1
@@ -1,9 +1,10 @@
|
||||
import type { HypothesisMetric, JevJudgment, Json, Submission } from "../vocab/index.ts";
|
||||
import { questionsOf, taskOf, type Question } from "./questions.ts";
|
||||
|
||||
// 问 Jev:从带类型的输入拼材料、发请求、读回答(docs/方法.md 4.3 ③⑥⑨)。
|
||||
// 问 Jev:从带类型的输入拼材料、发请求、读回答(docs/方法.md 4.3 ③⑥⑨ 和诊 1、诊 2)。
|
||||
// 材料里放什么、字段叫什么只在这里定。分请求照参考项目 src/judge/judge.ts:
|
||||
// 判重一对一次请求;冻结条款三问每问单独一次请求,材料相同;代码改动一个差异块一次请求,五问同在。
|
||||
// 诊断实验的判重、冻结检查照判重、冻结条款的写法;诊断程序的代码改动直接用 checkHunk。
|
||||
|
||||
type Factor = Submission<"proposal">["factors"][number];
|
||||
|
||||
@@ -13,6 +14,12 @@ type Factor = Submission<"proposal">["factors"][number];
|
||||
*/
|
||||
export type DedupFactor = Pick<Factor, "name" | "method" | "keySettings">;
|
||||
|
||||
/** 判重看的诊断实验:名称、动哪一处、怎么动。能分开哪几条解释写在正文里,不影响"是不是同一个实验"。 */
|
||||
export type DedupExperiment = Pick<Submission<"diagnosisProposal">["experiments"][number], "name" | "spot" | "body">;
|
||||
|
||||
/** 冻结检查看的诊断实验:名称、动哪一处、怎么动,加登记的假设专用指标(编号是短名)。 */
|
||||
export type DiagnosisExperiment = DedupExperiment & Pick<Submission<"diagnosisProposal">["experiments"][number], "metrics">;
|
||||
|
||||
/** 一份冻结协议的条款(docs/方法.md 4.3 ⑥"冻结协议包括"),字段与统一提交相同。 */
|
||||
export type FrozenProtocol = Pick<Submission<"unification">, "factors" | "safeguards" | "metrics" | "body" | "clauses">;
|
||||
|
||||
@@ -49,6 +56,31 @@ export async function judgeSameChange(model: string, a: readonly DedupFactor[],
|
||||
return judgment as JevJudgment;
|
||||
}
|
||||
|
||||
/** 诊 2 判重:两个诊断实验是不是同一个实验。 */
|
||||
export async function judgeSameExperiment(model: string, a: DedupExperiment, b: DedupExperiment): Promise<JevJudgment> {
|
||||
const state = {
|
||||
task: taskOf("diagnosisDedup"),
|
||||
experiment_a: dedupExperiment(a),
|
||||
experiment_b: dedupExperiment(b),
|
||||
};
|
||||
const [judgment] = await ask(model, state, questionsOf("diagnosisDedup"));
|
||||
return judgment as JevJudgment;
|
||||
}
|
||||
|
||||
/** 诊 1 查一个诊断实验:改动了评测、自相矛盾、写得太含糊。三问逐个发,材料相同。 */
|
||||
export async function checkExperiment(model: string, experiment: DiagnosisExperiment): Promise<JevCheck> {
|
||||
const state = {
|
||||
task: taskOf("diagnosisFreeze"),
|
||||
experiment: {
|
||||
...dedupExperiment(experiment),
|
||||
metrics: experiment.metrics.map((m) => ({ id: m.id, howMeasured: m.howMeasured, missing: m.missing })),
|
||||
},
|
||||
};
|
||||
const judgments: JevJudgment[] = [];
|
||||
for (const question of questionsOf("diagnosisFreeze")) judgments.push(...(await ask(model, state, [question])));
|
||||
return check(judgments);
|
||||
}
|
||||
|
||||
/** ⑥ 查一份冻结协议的条款:改动了评测、自相矛盾、写得太含糊。三问逐个发,材料相同。 */
|
||||
export async function checkProtocol(model: string, protocol: FrozenProtocol): Promise<JevCheck> {
|
||||
const state = {
|
||||
@@ -83,6 +115,10 @@ function dedupFactor(f: DedupFactor): { [key: string]: Json } {
|
||||
return { name: f.name, method: f.method, keySettings: f.keySettings };
|
||||
}
|
||||
|
||||
function dedupExperiment(e: DedupExperiment): { [key: string]: Json } {
|
||||
return { name: e.name, spot: e.spot, body: e.body };
|
||||
}
|
||||
|
||||
function check(judgments: JevJudgment[]): JevCheck {
|
||||
return { flagged: judgments.some((j) => j.yes), judgments };
|
||||
}
|
||||
|
||||
+136
-3
@@ -1,13 +1,17 @@
|
||||
import { hash, type Json } from "../vocab/index.ts";
|
||||
|
||||
// Jev 的九问(docs/方法.md 4.3 ③⑥⑨):每一处的材料说明、问题原文、阈值。
|
||||
// Jev 的十三问(docs/方法.md 4.3 ③⑥⑨ 和诊 1、诊 2):每一处的材料说明、问题原文、阈值。
|
||||
// 措辞、判"是"判"否"的说明和示例句取自参考项目 src/judge/questions.ts(填圆那一套),
|
||||
// 只改了必须改的:点 → 因子,字段名对上 vocab 的提案和冻结协议,改动对上 workspace 的比较结果。
|
||||
// 写法照参考项目 research-wiki/参考-Jev使用指南.md:问题用英文、一问只问一件事、
|
||||
// 统一问坏的一面(概率高 = 有问题或"是同一个")、判据带 not_for 和例子、问题里用反引号点到材料路径。
|
||||
|
||||
/** 用到 Jev 的三处:判重、冻结条款、一段代码改动。 */
|
||||
export type Site = "dedup" | "freeze" | "diff";
|
||||
/**
|
||||
* 用到 Jev 的五处:判重、冻结条款、一段代码改动,以及诊断实验的判重和冻结检查。
|
||||
* 诊断的两处材料是"一个诊断实验"(动当前基线的一处),不是候选提案和冻结协议,所以各有自己的材料说明和问法
|
||||
* (方法.md 4.3 诊 1、诊 2);一段代码改动的五问直接沿用 diff(诊 4)。
|
||||
*/
|
||||
export type Site = "dedup" | "freeze" | "diff" | "diagnosisDedup" | "diagnosisFreeze";
|
||||
|
||||
/** 一条是非题(Jev 的 noul)。 */
|
||||
export interface Noul {
|
||||
@@ -24,6 +28,9 @@ const THRESHOLDS: Readonly<Record<Site, Readonly<Record<string, number>>>> = {
|
||||
dedup: { same_change: 0.3 },
|
||||
freeze: { self_contradictory: 0.5, changes_measurement: 0.21, too_vague: 0.78 },
|
||||
diff: { unseeded_randomness: 0.55, reads_clock: 0.55, touches_measurement: 0.65, early_exit: 0.63, zero_for_unmeasured: 0.62 },
|
||||
// 诊断实验的两处阈值暂用 dedup、freeze 的值,没有在诊断实验的材料上校准过,等真实数据(任务书 18 第三块)再校准。
|
||||
diagnosisDedup: { same_experiment: 0.3 },
|
||||
diagnosisFreeze: { self_contradictory: 0.5, changes_measurement: 0.21, too_vague: 0.78 },
|
||||
};
|
||||
|
||||
/** 放进材料 `task` 字段的说明:这是什么材料、各部分是什么。 */
|
||||
@@ -44,6 +51,18 @@ const TASKS: Readonly<Record<Site, string>> = {
|
||||
"`protocol.metrics` lists the metrics the protocol requires the implementer " +
|
||||
"to compute in the candidate code and write to metrics.json (each with its `howMeasured` and `missing` rule; an empty list means none). " +
|
||||
"Only this hunk is shown; judge only what the hunk itself adds or changes.",
|
||||
diagnosisDedup:
|
||||
"Two diagnosis experiments from the same round of a study that improves a program by changing its code. " +
|
||||
"A diagnosis experiment takes the current baseline program, changes ONE spot of it (`spot`) as `body` describes " +
|
||||
"(replace a step with a clearly stronger version, replace it with another method, remove it, or relax a restriction), " +
|
||||
"runs the changed program once and scores the answer it returns. " +
|
||||
"The experiments were written by different researchers, in Chinese or English.",
|
||||
diagnosisFreeze:
|
||||
"One diagnosis experiment in a study that improves a program by changing its code. " +
|
||||
"An implementer will change the baseline program at `experiment.spot` exactly as `experiment.body` says and nothing else; " +
|
||||
"the unchanged task harness will then run the changed program once, and the task's validator will score the answer it returns. " +
|
||||
"The implementer may not edit the task's harness or validator files. " +
|
||||
"`experiment.metrics` lists extra diagnostic metrics the changed program must record (an empty list means none).",
|
||||
};
|
||||
|
||||
const QUESTIONS: Readonly<Record<Site, Readonly<Record<string, Noul>>>> = {
|
||||
@@ -292,6 +311,120 @@ const QUESTIONS: Readonly<Record<Site, Readonly<Record<string, Noul>>>> = {
|
||||
},
|
||||
},
|
||||
},
|
||||
// 诊 2 诊断实验判重:一对实验一次请求,写法照 dedup。
|
||||
diagnosisDedup: {
|
||||
same_experiment: {
|
||||
type: "noul",
|
||||
instructions: {
|
||||
question:
|
||||
"Do `experiment_a` and `experiment_b` change the same spot of the baseline in the same way, so that one run would answer both?",
|
||||
focus:
|
||||
"Compare where each experiment changes the baseline (`spot`) and how (`body`): the kind of change (stronger version, other method, removal, relaxed restriction) and its settings. " +
|
||||
"Ignore wording, language, level of detail, the experiment's name, and which explanation it says it can separate from which. " +
|
||||
"The same spot changed in a different way, or the same change with different settings or sizes, is NOT the same experiment: each would need its own run.",
|
||||
},
|
||||
criteria: {
|
||||
true: {
|
||||
what: "Both change the same spot of the baseline in the same way with the same settings; they differ only in wording, language or detail",
|
||||
examples: [
|
||||
"'replace the optimizer step with an exact solver, no time limit' vs 'swap the optimizer for an exact solve without any time cap'",
|
||||
"'remove the final repair pass' vs 'delete the repair pass at the end of the pipeline'",
|
||||
],
|
||||
},
|
||||
false: {
|
||||
what: "They change different spots, change the same spot in a different way, or use the same change with different settings",
|
||||
not_for: "Two experiments that merely test the same explanation, or merely touch the same function at different spots",
|
||||
examples: [
|
||||
"'remove the repair pass' vs 'replace the repair pass with an exact solver'",
|
||||
"'run the search for 10x more steps' vs 'run the search for 100x more steps'",
|
||||
"'relax the iteration limit of the search' vs 'relax the time limit of the whole program'",
|
||||
],
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
|
||||
// 诊 1 诊断实验冻结检查:每问单独一次请求,材料相同;问法照 freeze,材料换成"一个诊断实验"。
|
||||
diagnosisFreeze: {
|
||||
self_contradictory: {
|
||||
type: "noul",
|
||||
instructions: {
|
||||
question:
|
||||
"Does `experiment` ask for something that cannot be done as written: do two of its statements exclude each other, or does `experiment.body` change places other than the one `experiment.spot` names?",
|
||||
focus:
|
||||
"Take the change described in `experiment.body` and check it against `experiment.spot`: the experiment must change that one spot and keep everything else of the baseline as it is. " +
|
||||
"A change that cannot be carried out without also changing something else, or statements that require things that cannot both hold, make it contradictory. " +
|
||||
"A statement that is merely unusual, ambitious or likely to give a worse score is not a contradiction.",
|
||||
},
|
||||
criteria: {
|
||||
true: {
|
||||
what: "Carrying out the experiment necessarily changes more than the named spot, or two statements require things that cannot both hold",
|
||||
examples: [
|
||||
"`spot` names the radius computation while `body` also asks to replace the starting layout",
|
||||
"`body` says remove the step and also tune its step size",
|
||||
"`body` says change only the solver and also keep the solver's code unchanged",
|
||||
],
|
||||
},
|
||||
false: {
|
||||
what: "All statements can hold at the same time in one implementation that changes only the named spot",
|
||||
not_for: "Experiments that may not improve the score, or statements that are vague but not conflicting",
|
||||
examples: ["replace the solver with an exact LP and keep everything else as in the baseline", "remove the final repair pass"],
|
||||
},
|
||||
},
|
||||
},
|
||||
changes_measurement: {
|
||||
type: "noul",
|
||||
instructions: {
|
||||
question:
|
||||
"Does `experiment` require changing how the answer is scored or checked, rather than changing the solution that gets scored?",
|
||||
focus:
|
||||
"The answer of the changed program is scored by the task's validator and harness. " +
|
||||
"Changing the algorithm that produces the answer is allowed, including relaxing a limit or a check inside the program itself, as long as the returned answer is still scored as before. " +
|
||||
"Changing how the returned value is summed, scored, validated, rounded or reported, or the validator or harness themselves, is a change of measurement. " +
|
||||
"Rules for the experiment's own diagnostic metrics (what they count, how they are aggregated) do not change the score.",
|
||||
},
|
||||
criteria: {
|
||||
true: {
|
||||
what: "The experiment asks to alter the scoring or validation: the formula of the score, what gets counted, tolerances, the harness or validator, or the value reported instead of the real one",
|
||||
examples: [
|
||||
"report the sum of diameters instead of the sum of radii",
|
||||
"count only items above a size threshold when summing",
|
||||
"raise the overlap tolerance in the validator",
|
||||
],
|
||||
},
|
||||
false: {
|
||||
what: "The experiment only changes how the answer is built or searched; the score is computed from its output as before",
|
||||
not_for: "Adding diagnostic metrics that are recorded next to the score without replacing it",
|
||||
examples: ["replace the greedy step with an exact solve", "remove the repair pass and see what the raw answer scores"],
|
||||
},
|
||||
},
|
||||
},
|
||||
too_vague: {
|
||||
type: "noul",
|
||||
instructions: {
|
||||
question:
|
||||
"Does `experiment` leave the implementer to guess what to change, for example which spot of the baseline to edit, what to replace it with, or the settings of the stronger version?",
|
||||
focus:
|
||||
"Read `experiment.spot` and `experiment.body` as an implementer who must follow them without asking questions. " +
|
||||
"Brief text is fine when it names the spot, the kind of change and its settings. " +
|
||||
"Judge only whether each statement is specific; whether two statements conflict is checked separately.",
|
||||
},
|
||||
criteria: {
|
||||
true: {
|
||||
what: "An implementer would have to invent a key part of the experiment: the spot to change, what to put there, or its main settings",
|
||||
examples: ["make the search better somewhere", "replace the optimizer with a stronger one", "relax some of the limits"],
|
||||
},
|
||||
false: {
|
||||
what: "The spot, the kind of change and its settings are stated concretely enough to implement directly",
|
||||
not_for: "Experiments that leave only minor coding details, such as variable names, to the implementer",
|
||||
examples: [
|
||||
"in the radius step, replace the heuristic with an exact LP solved by scipy linprog; no time limit",
|
||||
"remove the final repair pass and return the raw answer",
|
||||
],
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
};
|
||||
|
||||
/** 一问的全部内容:问题原文、阈值,以及材料说明、问题原文、阈值三者合起来的哈希。 */
|
||||
|
||||
+7
-5
@@ -17,7 +17,7 @@ import {
|
||||
type SubmissionKind,
|
||||
} from "../vocab/index.ts";
|
||||
|
||||
// 账本记录的种类(docs/结构.md 第四部分的 11 种记录,细分成下面 19 种),每种一份内容定义(TypeBox)。
|
||||
// 账本记录的种类(docs/结构.md 第四部分的 11 种记录,细分成下面 20 种),每种一份内容定义(TypeBox)。
|
||||
// 下面写"只有一条""只一次"的,由 schema.ts 的唯一约束保证。
|
||||
// 配置、冻结协议、决定、结算的内容由别的模块产生,这里只检查最少的标识字段,其余只检查是 Json。
|
||||
|
||||
@@ -131,7 +131,7 @@ export const KINDS = {
|
||||
submission: Type.Unsafe<StoredSubmission>({}),
|
||||
/**
|
||||
* 3 退回:被退回的那份内容原样存(它可能正是因为不合格式才被退回),连同退回理由。
|
||||
* target 是这次任务的对象,与提交的 target 同义(候选、原提案、消融格,没有对象的写 null):
|
||||
* target 是这次任务的对象,与提交的 target 同义(候选、原提案、消融格、本轮诊断,没有对象的写 null):
|
||||
* 同一人在同一阶段可能对几个对象各做一次同种任务,重新发起时按它找本人这个对象的上一次退回。
|
||||
* jev 是这次退回用到的 Jev 判断(见 JevUsed),判"否"的几问也一起存;没用到 Jev 的写 null。
|
||||
*/
|
||||
@@ -161,11 +161,11 @@ export const KINDS = {
|
||||
/** 6 冻结代码:候选或消融格的代码版本号(由 workspace 给出)和因子位置表。位置表先存 Json。 */
|
||||
code: Type.Object({ object: Id, version: Text, locations: AnyJson }, CLOSED),
|
||||
|
||||
/** 7 运行·登记要跑(占名额):运行编号由 flow 起,一个编号只登记一次;用途、轮次、对象(候选或格;基线没有)、运行定义。 */
|
||||
/** 7 运行·登记要跑(占名额):运行编号由 flow 起,一个编号只登记一次;用途、轮次、对象(候选、格或本轮诊断;基线没有)、运行定义。 */
|
||||
runPlanned: Type.Object(
|
||||
{
|
||||
run: Id,
|
||||
purpose: Type.Enum(["baseline", "forward", "ablation"], { type: "string" }),
|
||||
purpose: Type.Enum(["baseline", "forward", "ablation", "diagnosis"], { type: "string" }),
|
||||
round: Round,
|
||||
object: OptionalId,
|
||||
definition: RunDefinition,
|
||||
@@ -178,7 +178,7 @@ export const KINDS = {
|
||||
runResult: Type.Object({ run: Id, process: ProcessResult, judged: Type.Union([JudgedResult, Type.Null()]) }, CLOSED),
|
||||
|
||||
/**
|
||||
* 8 决定:选批、采纳、选格、押的因子、入口分配、判重合并。decision 是决定种类。
|
||||
* 8 决定:选诊断实验、选批、采纳、选格、押的因子、入口分配、判重合并(提案和诊断实验)。decision 是决定种类。
|
||||
* 输入输出存 Json:ledger 不认识 rules。判重合并的输入里带 Jev 的原始概率。
|
||||
*/
|
||||
decision: Type.Object(
|
||||
@@ -197,6 +197,8 @@ export const KINDS = {
|
||||
|
||||
/** 9 结算:每轮最多采纳一个改法,所以最多一次消融,结算按轮次一轮一条。各格结局、配合关系、解释的确认或否定等先存 Json。 */
|
||||
settlement: Type.Object({ round: Round, result: AnyJson }, CLOSED),
|
||||
/** 9 诊断结算:每轮最多一次诊断,结算也按轮次一轮一条(方法.md 6.12)。每个实验的结局与实测、解释的确认或否定等先存 Json。 */
|
||||
diagnosisSettlement: Type.Object({ round: Round, result: AnyJson }, CLOSED),
|
||||
|
||||
/** 10 模型调用开始:先登记再调用(方法.md 第九部分)。附这次任务的全部材料,用来重现模型当时看到了什么。一次调用只有一条。 */
|
||||
callStarted: Type.Object({ call: Id, actor: Id, stageId: OptionalId, model: Text, materials: AnyJson }, CLOSED),
|
||||
|
||||
+13
-5
@@ -83,7 +83,7 @@ export class Ledger {
|
||||
}
|
||||
|
||||
/**
|
||||
* 追加一条记录,返回存下的那一条。先按种类检查内容,再做写入时的四种拒收检查(rejections.ts)。
|
||||
* 追加一条记录,返回存下的那一条。先按种类检查内容,再做写入时的五种拒收检查(rejections.ts)。
|
||||
* 写提案时,作者自评从提案里拆出来,在同一个事务里另存成一条封存,返回的是提案那一条。
|
||||
*/
|
||||
append<K extends AppendKind>(kind: K, content: Input<K>): LedgerRecord<K> {
|
||||
@@ -117,16 +117,24 @@ export class Ledger {
|
||||
this.#first("stageOpen", "json_extract(content, '$.stageId') = ?", stageId) as LedgerRecord<"stageOpen"> | undefined,
|
||||
latest: (slot) => this.latestSubmission(slot),
|
||||
runStarted: (run) => this.#first("runStarted", "json_extract(content, '$.run') = ?", run) !== undefined,
|
||||
ablationStarted: (round) =>
|
||||
ablationStarted: (round) => this.#purposeStarted("ablation", round),
|
||||
diagnosisStarted: (round) => this.#purposeStarted("diagnosis", round),
|
||||
};
|
||||
|
||||
/** 这一轮有没有某种用途的运行已经确认启动。 */
|
||||
#purposeStarted(purpose: "ablation" | "diagnosis", round: number): boolean {
|
||||
return (
|
||||
this.#one(
|
||||
`SELECT s.seq FROM records s JOIN records p
|
||||
ON p.kind = 'runPlanned' AND json_extract(p.content, '$.run') = json_extract(s.content, '$.run')
|
||||
WHERE s.kind = 'runStarted'
|
||||
AND json_extract(p.content, '$.purpose') = 'ablation'
|
||||
AND json_extract(p.content, '$.purpose') = ?
|
||||
AND json_extract(p.content, '$.round') = ?`,
|
||||
purpose,
|
||||
round,
|
||||
) !== undefined,
|
||||
};
|
||||
) !== undefined
|
||||
);
|
||||
}
|
||||
|
||||
// -------------------------------------------------------------------------
|
||||
// 读取
|
||||
|
||||
@@ -1,8 +1,8 @@
|
||||
import { SUBMISSION_KINDS, type Stage } from "../vocab/index.ts";
|
||||
import type { Content, Kind, LedgerRecord, SubmissionSlot } from "./kinds.ts";
|
||||
|
||||
// 写入时由 ledger 拒绝的四种情况(docs/结构.md 第四部分"写入时由 ledger 拒绝的几种情况"),在写入的同一个事务里查。
|
||||
// 只有这四种。记录之间别的先后(例如运行结果前要先有登记)和提交种类与阶段对不对得上都不查:
|
||||
// 写入时由 ledger 拒绝的五种情况(docs/结构.md 第四部分"写入时由 ledger 拒绝的几种情况"),在写入的同一个事务里查。
|
||||
// 只有这五种。记录之间别的先后(例如运行结果前要先有登记)和提交种类与阶段对不对得上都不查:
|
||||
// flow 调 ledger 是模块之间的调用,不是系统边界。"只发生一次"由 schema.ts 的唯一约束管。
|
||||
// 提交内容的字段归 submissions,该不该开下一个阶段归 flow。
|
||||
|
||||
@@ -16,14 +16,18 @@ export interface WriteState {
|
||||
runStarted(run: string): boolean;
|
||||
/** 这一轮有没有消融格的运行已经确认启动。 */
|
||||
ablationStarted(round: number): boolean;
|
||||
/** 这一轮的诊断运行有没有已经确认启动。 */
|
||||
diagnosisStarted(round: number): boolean;
|
||||
}
|
||||
|
||||
/** 实现和核验阶段的提交不能替换(方法.md 第五部分总原则):实现、修复、核验、消融实现、消融核验。 */
|
||||
/** 实现和核验阶段的提交不能替换(方法.md 第五部分总原则):实现、修复、核验、消融实现、消融核验、诊断实现(含修复)、诊断核验。 */
|
||||
const NOT_REPLACEABLE: ReadonlySet<Stage> = new Set([
|
||||
"implementation",
|
||||
"verification",
|
||||
"ablationImplementation",
|
||||
"ablationVerification",
|
||||
"diagnosisImplementation",
|
||||
"diagnosisVerification",
|
||||
]);
|
||||
|
||||
export function checkWrite(kind: Kind, content: unknown, state: WriteState): void {
|
||||
@@ -35,7 +39,7 @@ export function checkWrite(kind: Kind, content: unknown, state: WriteState): voi
|
||||
/**
|
||||
* 第 1 条:只收开着的阶段的提交,即有开阶段记录、没有关阶段记录。
|
||||
* 第 2 条:新的一版必须正好接在本人在这个阶段、对同一个对象的最新一版后面(第一版是 1);
|
||||
* 实现、修复、核验、消融实现、消融核验只能交第一版。
|
||||
* 实现、修复、核验、消融实现、消融核验、诊断实现、诊断修复、诊断核验只能交第一版。
|
||||
*/
|
||||
function checkSubmission(s: Content<"submission">, state: WriteState): void {
|
||||
const where = `提交 ${s.kind}(阶段 ${s.stageId},作者 ${s.author},对象 ${s.target ?? "无"},编号 ${s.item ?? "无"},第 ${s.version} 版)`;
|
||||
@@ -63,7 +67,10 @@ function checkRunResult(r: Content<"runResult">, state: WriteState): void {
|
||||
}
|
||||
}
|
||||
|
||||
/** 第 4 条:这一轮已经有消融格确认启动了,就不能再关解释阶段(方法.md 4.3 ⑭:解释冻结必须早于任何一格启动)。 */
|
||||
/**
|
||||
* 第 4 条:这一轮已经有消融格确认启动了,就不能再关解释阶段(方法.md 4.3 ⑭:解释冻结必须早于任何一格启动)。
|
||||
* 第 5 条:这一轮的诊断运行已经确认启动了,就不能再关诊断预测阶段(方法.md 4.3 诊 2:冻结必须早于诊断运行启动,范围才算事前预测)。
|
||||
*/
|
||||
function checkStageClose(c: Content<"stageClose">, state: WriteState): void {
|
||||
const open = state.stageOpen(c.stageId);
|
||||
if (open?.content.stage === "explanation" && state.ablationStarted(open.content.round)) {
|
||||
@@ -71,4 +78,9 @@ function checkStageClose(c: Content<"stageClose">, state: WriteState): void {
|
||||
`不能关解释阶段 ${c.stageId}:第 ${open.content.round} 轮已经有消融格确认启动,解释冻结必须早于任何一格启动(方法.md 4.3 ⑭)`,
|
||||
);
|
||||
}
|
||||
if (open?.content.stage === "diagnosisPrediction" && state.diagnosisStarted(open.content.round)) {
|
||||
throw new Error(
|
||||
`不能关诊断预测阶段 ${c.stageId}:第 ${open.content.round} 轮的诊断运行已经确认启动,诊断预测冻结必须早于诊断运行启动(方法.md 4.3 诊 2)`,
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -3,7 +3,7 @@ import type { DatabaseSync } from "node:sqlite";
|
||||
// 账本的表结构(docs/结构.md 第四部分"全部只追加"):只有一张表,每一行是序号、种类、内容、内容的哈希。
|
||||
// 表结构一改就把版本号加 1;旧文件版本号对不上时打开报错,不迁移,也不逐字核对建表语句。
|
||||
|
||||
export const SCHEMA_VERSION = 4;
|
||||
export const SCHEMA_VERSION = 5;
|
||||
|
||||
/** 新文件要执行的建表语句,由 Ledger 在一个事务里执行,最后写入版本号。 */
|
||||
export const CREATE_SCHEMA = [
|
||||
@@ -33,6 +33,8 @@ export const CREATE_SCHEMA = [
|
||||
`CREATE UNIQUE INDEX call_ended_once ON records (json_extract(content, '$.call')) WHERE kind = 'callEnded'`,
|
||||
// 结算以轮次为编号:每轮最多采纳一个改法,所以最多一次消融。
|
||||
`CREATE UNIQUE INDEX settlement_once ON records (json_extract(content, '$.round')) WHERE kind = 'settlement'`,
|
||||
// 诊断结算同样以轮次为编号:每轮最多一次诊断。
|
||||
`CREATE UNIQUE INDEX diagnosis_settlement_once ON records (json_extract(content, '$.round')) WHERE kind = 'diagnosisSettlement'`,
|
||||
`PRAGMA user_version=${SCHEMA_VERSION}`,
|
||||
];
|
||||
|
||||
|
||||
@@ -3,7 +3,14 @@ import { type Draft, factorLines, KIND_TEXT, offText, registeredMetrics, taskPoi
|
||||
import type { Board } from "./read.ts";
|
||||
import { fenced, json } from "./format.ts";
|
||||
import { unassignedChanges } from "./locations.ts";
|
||||
import { clauseKey, factorKey, splitCell, type FrozenProtocol, type LocationTable } from "./shapes.ts";
|
||||
import {
|
||||
clauseKey,
|
||||
diagnosisId,
|
||||
factorKey,
|
||||
splitCell,
|
||||
type FrozenProtocol,
|
||||
type LocationTable,
|
||||
} from "./shapes.ts";
|
||||
|
||||
// 实验员、核验员的 5 种任务(方法.md 4.3 ⑨ ⑯):只看分给他的那个候选或那一格——冻结协议、起点代码、
|
||||
// 要实现或核验的东西。不给信念本(用户 2026-10-03 决定 3)、不给任何运行成绩、不给别的候选。
|
||||
@@ -36,8 +43,7 @@ export function protocolMetrics(board: Board, p: FrozenProtocol): HypothesisMetr
|
||||
})];
|
||||
}
|
||||
|
||||
function metricSection(d: Draft, p: FrozenProtocol): void {
|
||||
const all = protocolMetrics(d.board, p);
|
||||
function metricSection(d: Draft, all: HypothesisMetric[]): void {
|
||||
d.section("要实现的指标定义", [
|
||||
all.length === 0 ? "(这份协议没有要实现的假设专用指标)" : json(all),
|
||||
"",
|
||||
@@ -58,15 +64,22 @@ function codeLines(d: Draft): string[] {
|
||||
return [...d.readableLines(), "读这些目录用 read_source(核验员)或 shell(实验员)。"];
|
||||
}
|
||||
|
||||
/** 本轮做过诊断时,诊断程序的代码也能读(只读),冻结协议可以写"照诊断实验 dN 的做法"(方法.md 4.3 ②);只给代码,不给成绩和实验原文。 */
|
||||
function diagnosisReference(d: Draft): string[] {
|
||||
if (d.board.diagnosisSettlement(d.round) === undefined) return [];
|
||||
d.read(diagnosisId(d.round), "本轮诊断程序(只读,冻结协议引用诊断实验时作参考)");
|
||||
return ["上面还列了本轮诊断程序的代码:冻结协议引用诊断实验时作参考,别把它当成这个候选的起点。"];
|
||||
}
|
||||
|
||||
/** ⑨ 实现:冻结协议、指标定义、要交的位置表、起点代码(本批基线)。 */
|
||||
export function implementation(d: Draft): ChecklistOf<"implementation"> {
|
||||
const candidate = d.req.object!;
|
||||
const { p, items } = frozen(d, candidate);
|
||||
taskSection(d, "实验员", `候选 ${candidate}`);
|
||||
d.read(p.baseline, "起点代码(本批基线)");
|
||||
d.section("基线代码", [...codeLines(d), "你的工作目录是起点代码的副本,改动都落在工作目录里。"]);
|
||||
d.section("基线代码", [...codeLines(d), "你的工作目录是起点代码的副本,改动都落在工作目录里。", ...diagnosisReference(d)]);
|
||||
d.section("本任务要处理的东西", [`### 候选 ${candidate} 的冻结协议`, "", json(p)]);
|
||||
metricSection(d, p);
|
||||
metricSection(d, protocolMetrics(d.board, p));
|
||||
locationTableSection(d, p);
|
||||
return { kind: "implementation", ...items };
|
||||
}
|
||||
@@ -78,7 +91,7 @@ export function repair(d: Draft): ChecklistOf<"repair"> {
|
||||
taskSection(d, "实验员", `候选 ${candidate}`);
|
||||
d.read(p.baseline, "起点代码(本批基线)");
|
||||
d.read(candidate, "你上一次提交的代码");
|
||||
d.section("基线代码", [...codeLines(d), "你的工作目录是上一次提交的代码的副本。"]);
|
||||
d.section("基线代码", [...codeLines(d), "你的工作目录是上一次提交的代码的副本。", ...diagnosisReference(d)]);
|
||||
const v = d.board.published("verification").filter((s) => s.target === candidate).at(-1)!.content;
|
||||
d.section("本任务要处理的东西", [
|
||||
"### 退回意见",
|
||||
@@ -102,7 +115,7 @@ export function repair(d: Draft): ChecklistOf<"repair"> {
|
||||
"",
|
||||
json(p),
|
||||
]);
|
||||
metricSection(d, p);
|
||||
metricSection(d, protocolMetrics(d.board, p));
|
||||
locationTableSection(d, p);
|
||||
return { kind: "repair", ...items };
|
||||
}
|
||||
@@ -148,7 +161,7 @@ export function verification(d: Draft): ChecklistOf<"verification"> {
|
||||
"",
|
||||
fenced(d.req.changes!.diff, "diff"),
|
||||
]);
|
||||
metricSection(d, p);
|
||||
metricSection(d, protocolMetrics(d.board, p));
|
||||
return { kind: "verification", unassigned: found.unassigned.map((u) => u.location), ...items };
|
||||
}
|
||||
|
||||
@@ -212,3 +225,60 @@ export function ablationVerification(d: Draft): ChecklistOf<"ablationVerificatio
|
||||
]);
|
||||
return { kind: "ablationVerification", cell: a.cell, factors: a.factors };
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// 诊断的三种任务(方法.md 4.3 诊 4):对象是本轮诊断,一位实验员把选中的全部实验写进一份诊断程序,一位核验员逐个核对。
|
||||
// 只看选中的实验原文和当前基线代码,不给信念本、不给任何成绩。
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
/** 三种诊断任务共用:任务与本轮、起点代码(当前基线)、选中的实验原文和指标定义;program 是另外要能读的诊断程序的说明。交回选中的实验编号。 */
|
||||
function diagnosisTask(d: Draft, role: string, program: string | null): string[] {
|
||||
d.group("分给你的材料", "下面是分给你的本轮诊断的全部材料。");
|
||||
const chosen = d.board.decision("selectDiagnosis", d.round)!.output.chosen;
|
||||
const experiments = d.board.diagnosisExperiments(d.round).filter((e) => chosen.includes(e.id));
|
||||
taskSection(d, role, `本轮诊断(对象 ${diagnosisId(d.round)})`);
|
||||
d.read(d.board.baselineOf(d.round), "起点代码(当前基线)");
|
||||
if (program !== null) d.read(diagnosisId(d.round), program);
|
||||
d.section("基线代码", codeLines(d));
|
||||
d.section("选中的诊断实验", [
|
||||
"每个实验都从当前基线的代码出发,只动它写的那一处,其余照基线;每个实验交出自己的答案。",
|
||||
"",
|
||||
...experiments.flatMap((e) => [`### ${e.id} ${e.name}`, "", `动哪一处:${e.spot}`, "", e.body, ""]),
|
||||
]);
|
||||
metricSection(d, experiments.flatMap((e) => e.metrics));
|
||||
return chosen;
|
||||
}
|
||||
|
||||
/** 诊断实现,加上原始产物的位置;修复(repair 为 true)另放退回意见(没通过的实验和依据、上一次的位置表和改动),通过的实验不要动。 */
|
||||
function diagnosisCode(d: Draft, repair: boolean): string[] {
|
||||
const chosen = diagnosisTask(d, "实验员", repair ? "你上一次提交的诊断程序" : null);
|
||||
d.section("原始产物", [
|
||||
`产出当前基线的那次正式运行(${d.board.runOf(d.board.baselineOf(d.round))!.plan.run})的原始产物已经只读地放进你的工作目录,实验要接着算的答案就在里面;位置和怎么读写在任务说明里(research_query 查 task)。`,
|
||||
"你的工作目录是当前基线代码的副本,加上这份原始产物,改动都落在工作目录里。",
|
||||
]);
|
||||
if (!repair) return chosen;
|
||||
const v = d.board.published("diagnosisVerification").filter((s) => s.round === d.round).at(-1)!.content;
|
||||
const failed = v.experiments.filter((x) => !x.followed || !x.nothingElse);
|
||||
d.section("退回意见", [
|
||||
...failed.map((x) => `- 实验 ${x.experiment}:${x.followed ? "" : "没照写法做;"}${x.nothingElse ? "" : "动了写法以外的地方;"}${x.where}`),
|
||||
"",
|
||||
`核验员的说明:${v.body}`,
|
||||
"只修上面没通过的实验,通过的实验的代码不要动。",
|
||||
"",
|
||||
...["### 上一次交的位置表", "", json(d.board.code(diagnosisId(d.round)).locations), "", "### 上一次提交相对起点代码的改动", "", fenced(d.req.changes!.diff, "diff")],
|
||||
]);
|
||||
return chosen;
|
||||
}
|
||||
|
||||
export const diagnosisImplementation = (d: Draft): ChecklistOf<"diagnosisImplementation"> => ({ kind: "diagnosisImplementation", experiments: diagnosisCode(d, false) });
|
||||
export const diagnosisRepair = (d: Draft): ChecklistOf<"diagnosisRepair"> => ({ kind: "diagnosisRepair", experiments: diagnosisCode(d, true) });
|
||||
|
||||
/** 诊断核验:选中的实验原文、当前基线到诊断程序的改动、位置表。Jev 的判断不给核验员(同正向核验)。 */
|
||||
export function diagnosisVerification(d: Draft): ChecklistOf<"diagnosisVerification"> {
|
||||
const chosen = diagnosisTask(d, "核验员", "要核验的诊断程序");
|
||||
d.section("本任务要处理的东西", [
|
||||
...["### 实验员交的位置表", "", json(d.board.code(diagnosisId(d.round)).locations), ""],
|
||||
...["### 改动全文(诊断程序相对当前基线,每段前后各 3 行上下文)", "", fenced(d.req.changes!.diff, "diff")],
|
||||
]);
|
||||
return { kind: "diagnosisVerification", experiments: chosen };
|
||||
}
|
||||
|
||||
@@ -12,6 +12,7 @@ import {
|
||||
type Phase,
|
||||
} from "../rules/index.ts";
|
||||
import type { ProcessResult, Range } from "../vocab/index.ts";
|
||||
import { diagnosisHistory } from "./diagnosis.ts";
|
||||
import { measured, num, pct, PLACEMENT_TEXT, rangeText, resourcesText, score, SYNERGY_TEXT, UNTESTED_TEXT } from "./format.ts";
|
||||
import type { StudyFacts } from "./inputs.ts";
|
||||
import { answerPathLines } from "./locations.ts";
|
||||
@@ -21,6 +22,8 @@ import { CELL_NOT_RUN, cellId, factorId, factorKey, ORIGINAL_BASELINE, type Sett
|
||||
// 信念本(方法.md 6.10):程序从已公开的记录现算,不存。数字全由 rules 的函数算,这里只挑出来写成字。
|
||||
// 只读这几样:已关阶段的提交、按 Board.runPublic 公开的运行、决定、结算、解封;封存、第一次预测、退回一律不读。
|
||||
|
||||
const PHASE_TEXT: Readonly<Record<Phase, string>> = { forward: "正向", diagnosis: "诊断", ablation: "消融" };
|
||||
|
||||
/** 范围怎么换算成把握(方法.md 6.2、6.3),写在开头。只说程序怎么用这三个数,不要求研究者照着算。 */
|
||||
function rangeToBelief(lambda: number): string {
|
||||
return (
|
||||
@@ -118,6 +121,9 @@ export function beliefBookOf(board: Board, study: StudyFacts, round: number | nu
|
||||
}
|
||||
for (const s of settlements) lines.push(...ablationSummary(board, s));
|
||||
|
||||
// 3 诊断历史
|
||||
lines.push("", "**诊断历史**(每轮的诊断实验与实测、每条解释确认或否定、没做的实验和原因,以及程序推出的方向)", ...diagnosisHistory(board, study, latestRound));
|
||||
|
||||
// 4 战绩
|
||||
lines.push("", "**战绩**(只显示,不改任何权重;偏高偏低都按分数原值判)");
|
||||
const samples: Array<{ researcher: string; phase: Phase; range: Range; value: number }> = batches.flatMap((d) =>
|
||||
@@ -127,6 +133,9 @@ export function beliefBookOf(board: Board, study: StudyFacts, round: number | nu
|
||||
return v === null ? [] : c.ranges.map((r) => ({ researcher: r.researcher, phase: "forward" as const, range: r.range, value: v }));
|
||||
}),
|
||||
);
|
||||
for (const s of board.diagnosisSettlements())
|
||||
for (const e of s.input.experiments)
|
||||
if (e.outcome.kind === "valid") for (const x of s.input.explanations) samples.push({ researcher: x.author, phase: "diagnosis", range: x.experiments.find((y) => y.experiment === e.experiment)!.range, value: e.outcome.score });
|
||||
for (const s of settlements)
|
||||
for (const c of s.input.cells) {
|
||||
if (c.outcome.kind !== "valid") continue;
|
||||
@@ -139,7 +148,7 @@ export function beliefBookOf(board: Board, study: StudyFacts, round: number | nu
|
||||
if (tallies.length === 0) lines.push("- 还没有实测可以核对");
|
||||
for (const { researcher, phase, tally: t } of tallies) {
|
||||
lines.push(
|
||||
`- ${researcher},${phase === "forward" ? "正向" : "消融"} ${t.total} 次:中位数偏高 ${t.medianHigh}、偏低 ${t.medianLow};` +
|
||||
`- ${researcher},${PHASE_TEXT[phase]} ${t.total} 次:中位数偏高 ${t.medianHigh}、偏低 ${t.medianLow};` +
|
||||
`穿过上沿 ${t.aboveTop}、下沿 ${t.belowBottom};包住 ${t.covered};落点 ` +
|
||||
(Object.keys(PLACEMENT_TEXT) as Array<keyof typeof PLACEMENT_TEXT>).map((k) => `${PLACEMENT_TEXT[k]} ${t.placements[k]}`).join(","),
|
||||
);
|
||||
|
||||
+46
-4
@@ -2,11 +2,12 @@ import type { Ledger } from "../ledger/index.ts";
|
||||
import { adoptionLine, compareIds, runBalance } from "../rules/index.ts";
|
||||
import type { Checklist, SubmissionKind } from "../vocab/index.ts";
|
||||
import { beliefBookOf } from "./belief-book.ts";
|
||||
import { directionText, json, measured, score } from "./format.ts";
|
||||
import { diagnosisLines } from "./diagnosis.ts";
|
||||
import { directionText, json, measured, num, resourcesText, score } from "./format.ts";
|
||||
import type { TaskRequest } from "./inputs.ts";
|
||||
import { answerPathLines } from "./locations.ts";
|
||||
import { Board, type Run } from "./read.ts";
|
||||
import { entryId, explanationId, factorKey, splitCell, type FrozenProtocol } from "./shapes.ts";
|
||||
import { entryId, explanationId, factorKey, ORIGINAL_BASELINE, splitCell, type FrozenProtocol } from "./shapes.ts";
|
||||
|
||||
// 任务材料的形状,和研究者各任务共用的几块(方法.md 第三部分"给研究者的材料顺序")。
|
||||
|
||||
@@ -38,6 +39,11 @@ export const KIND_TEXT: Readonly<Record<SubmissionKind, string>> = {
|
||||
entry: "第零轮闭门写入口",
|
||||
discussion: "第零轮讨论",
|
||||
claim: "第零轮认领入口",
|
||||
diagnosisProposal: "写诊断解释与诊断实验",
|
||||
diagnosisPrediction: "诊断预测",
|
||||
diagnosisImplementation: "实现诊断实验",
|
||||
diagnosisRepair: "按核验意见修复诊断程序",
|
||||
diagnosisVerification: "核验诊断程序",
|
||||
proposal: "提案",
|
||||
firstPrediction: "第一次预测",
|
||||
critique: "挑刺",
|
||||
@@ -142,12 +148,42 @@ export function codeSection(d: Draft): void {
|
||||
d.read(d.board.baselineOf(d.round), "本批基线");
|
||||
for (const r of d.board.runs()) {
|
||||
if (r.plan.object !== null && r.plan.purpose !== "baseline" && d.board.runPublic(r)) {
|
||||
d.read(r.plan.object, `${r.plan.purpose === "forward" ? "候选" : "消融格"} ${r.plan.object} 的代码`);
|
||||
d.read(r.plan.object, r.plan.purpose === "diagnosis" ? `第 ${r.plan.round} 轮诊断程序的代码` : `${r.plan.purpose === "forward" ? "候选" : "消融格"} ${r.plan.object} 的代码`);
|
||||
}
|
||||
}
|
||||
d.section("基线", [...baselineLines(d), "", "能读的代码(用 read_source 读):", ...d.readableLines()]);
|
||||
}
|
||||
|
||||
/** 本轮诊断结果(方法.md 第三部分材料顺序第 3 条):从提案起的全部研究者任务都有,没诊断写原因;诊断任务自己读时这一段还没走完,不放。 */
|
||||
export function diagnosisSection(d: Draft): void {
|
||||
if (d.board.diagnosisDone(d.round)) d.section("本轮诊断结果", diagnosisLines(d.board, d.req.study, d.round, true));
|
||||
}
|
||||
|
||||
/** 运行的产物只放进查询,这里列编号。 */
|
||||
export function artifactSection(d: Draft, heading: string): void {
|
||||
const artifacts = d.req.artifacts!;
|
||||
const listed = Object.keys(artifacts)
|
||||
.sort(compareIds)
|
||||
.flatMap((run) => Object.entries(artifacts[run]!).map(([file, text]) => `- ${d.record(`${run}/${file}`, `运行 ${run} 的产物 ${file}`, text)}`));
|
||||
d.section(heading, ["用 research_query 按编号查全文:", ...(listed.length === 0 ? ["(没有)"] : listed)]);
|
||||
}
|
||||
|
||||
/** 本批基线那次运行的结果:分数、判分、其余指标、资源、任务的附加记录;原始基线没有位置表,不写答案路径。 */
|
||||
export function baselineRunSection(d: Draft): void {
|
||||
const base = d.board.baselineOf(d.round);
|
||||
const run = d.board.publicRunOf(base)!;
|
||||
const j = run.result!.judged!;
|
||||
const others = Object.keys(j.metrics).filter((k) => k !== d.req.study.finalMetric).sort(compareIds);
|
||||
d.section("本批基线那次运行的结果", [
|
||||
...(base === ORIGINAL_BASELINE
|
||||
? [`运行 ${run.plan.run}:任务给的原始代码;${j.verdict === "valid" ? "算数" : "不算数"}(${j.verdictReason});分数 ${measured(j.score, score)}`]
|
||||
: runLines(d, run)),
|
||||
resourcesText(run.result!.process, d.board.publicRunOf(ORIGINAL_BASELINE)!.result!.process, d.req.study.wallSeconds),
|
||||
...(others.length === 0 ? [] : [`其余指标:${others.map((k) => `${k} ${measured(j.metrics[k]!, num)}`).join(";")}`]),
|
||||
`任务的附加记录(含合法化,原样):${JSON.stringify(j.extra)}`,
|
||||
]);
|
||||
}
|
||||
|
||||
export function beliefSection(d: Draft): void {
|
||||
d.section("信念本", [beliefBookOf(d.board, d.req.study, d.round)]);
|
||||
}
|
||||
@@ -194,7 +230,13 @@ export function publicRecords(d: Draft): void {
|
||||
e.content.explanations.forEach((x, i) => d.record(explanationId(e.round, e.author, i + 1), `解释 ${explanationId(e.round, e.author, i + 1)}(${e.author})`, x));
|
||||
}
|
||||
for (const w of b.published("wrapUp")) d.record(`${w.stageId}.${w.author}`, `${w.author} 第 ${w.round} 轮的收尾`, w.content.body);
|
||||
for (const r of b.runs()) if (r.plan.purpose !== "baseline" && b.runPublic(r)) d.record(r.plan.run, `运行 ${r.plan.run}`, [...runLines(d, r), "", json(r.result!.judged)].join("\n"));
|
||||
for (const r of b.runs()) {
|
||||
if (r.plan.purpose !== "baseline" && r.plan.purpose !== "diagnosis" && b.runPublic(r)) d.record(r.plan.run, `运行 ${r.plan.run}`, [...runLines(d, r), "", json(r.result!.judged)].join("\n"));
|
||||
}
|
||||
// 诊断的完整结果:各人给的三个数(结算输入)不放,只有合并后的范围和逐个实验的判分。
|
||||
for (const { round, input: _input, ...settled } of b.diagnosisSettlements()) {
|
||||
d.record(`diagnosis-r${round}`, `第 ${round} 轮诊断的完整结果`, json({ explanations: b.diagnosisExplanations(round), experiments: b.diagnosisExperiments(round), ...settled }));
|
||||
}
|
||||
d.section("研究记录", [
|
||||
"已经公开的记录,一行一条;全文用 research_query 按编号查。",
|
||||
...d.records.map((r) => `- ${r.id}:${r.title}`),
|
||||
|
||||
@@ -12,6 +12,11 @@ const BUILDERS: { [K in SubmissionKind]: (d: Draft) => Extract<Checklist, { kind
|
||||
entry: researcher.entry,
|
||||
discussion: researcher.discussion,
|
||||
claim: researcher.claim,
|
||||
diagnosisProposal: researcher.diagnosisProposal,
|
||||
diagnosisPrediction: researcher.diagnosisPrediction,
|
||||
diagnosisImplementation: assigned.diagnosisImplementation,
|
||||
diagnosisRepair: assigned.diagnosisRepair,
|
||||
diagnosisVerification: assigned.diagnosisVerification,
|
||||
proposal: researcher.proposal,
|
||||
firstPrediction: researcher.firstPrediction,
|
||||
critique: researcher.critique,
|
||||
|
||||
@@ -0,0 +1,78 @@
|
||||
import { runBalance } from "../rules/index.ts";
|
||||
import { measured, num, rangeText, resourcesText, score } from "./format.ts";
|
||||
import type { StudyFacts } from "./inputs.ts";
|
||||
import type { Board, MergedExperiment } from "./read.ts";
|
||||
import { DIAGNOSIS_NOT_RUN, diagnosisId, ORIGINAL_BASELINE, type DiagnosisSettlement } from "./shapes.ts";
|
||||
|
||||
// 一轮诊断的结果写成文字(方法.md 4.3 诊 4、6.10 第 3 条),公共材料的"本轮诊断结果"和信念本的"诊断历史"共用。
|
||||
// 各人对诊断实验给的三个数不放进来,只有合并后的范围。
|
||||
|
||||
const ENDING = { finished: "跑完", crashed: "跑崩了", timeout: "超过分到的时间被杀", notRun: "整次运行先结束,没轮到" } as const;
|
||||
const SKIPPED = { budget: "一次诊断最多做 D 个实验,放不下", value: "信息价值不足以抵过成本" } as const;
|
||||
|
||||
/** 第 round 轮没有诊断的原因(方法.md 4.3 诊 3、诊 4)。 */
|
||||
function noDiagnosisReason(board: Board, study: StudyFacts, round: number): string {
|
||||
const decision = board.decision("selectDiagnosis", round);
|
||||
if (decision === undefined) {
|
||||
const balance = runBalance(study.totalRuns, board.runStatesBefore(round)).balance;
|
||||
return balance < 2 ? `余额只剩 ${balance} 次,不到 2 次,要留 1 次给本轮正向` : "配置关掉了诊断";
|
||||
}
|
||||
if (!decision.output.diagnose) return `信息价值减成本都不大于 0,最好的组合净值 ${num(Math.max(...decision.output.combos.map((c) => c.net)))}`;
|
||||
return `选中的实验核验都没通过、修复次数用完,一个都没跑:${board.eventReason(DIAGNOSIS_NOT_RUN, diagnosisId(round))}`;
|
||||
}
|
||||
|
||||
const signed = (x: number): string => (x > 0 ? `+${score(x)}` : score(x));
|
||||
|
||||
/** 一个合并后的实验:做了什么、实测、结局、测前合并范围、比特、惊讶、用时。 */
|
||||
function experimentLine(board: Board, study: StudyFacts, round: number, s: DiagnosisSettlement, e: MergedExperiment): string {
|
||||
const row = s.output.experiments.find((x) => x.experiment === e.id)!;
|
||||
const judged = s.judged.experiments.find((x) => x.id === e.id);
|
||||
const dropped = s.dropped.find((x) => x.experiment === e.id);
|
||||
const skipped = board.decision("selectDiagnosis", round)!.output.experiments.find((y) => y.experiment === e.id)!;
|
||||
const outcome = {
|
||||
valid: () => `实测 ${measured(judged!.score, score)},改进量 ${signed(row.gain!)}`,
|
||||
untested: () => `没做:${SKIPPED[skipped.reason!]}${skipped.bestCombo === null ? "" : `(含它的最好组合净值 ${num(skipped.bestCombo.net)})`}`,
|
||||
insufficient: () => `证据不足:${dropped === undefined ? `${ENDING[judged!.ending]};${judged!.verdictReason}` : `核验不通过,${dropped.reason}`}`,
|
||||
}[row.outcome]();
|
||||
const parts = [outcome, `测前合并范围 ${rangeText(row.merged)}`];
|
||||
if (row.worldBits !== null) parts.push(`世界比特 ${num(row.worldBits)}`);
|
||||
if (row.surprise !== null) parts.push(`惊讶 ${measured(row.surprise.bits, (x) => `${num(x)} 比特`)}`);
|
||||
if (judged !== undefined) {
|
||||
parts.push(`用时 ${measured(judged.wallSeconds, (x) => `${num(x)} 秒`)}`);
|
||||
const others = Object.keys(judged.metrics).filter((k) => k !== study.finalMetric).sort();
|
||||
if (others.length > 0) parts.push(`其余指标:${others.map((k) => `${k} ${measured(judged.metrics[k]!, num)}`).join(";")}`);
|
||||
}
|
||||
return `- ${e.id} ${e.name}(作者 ${e.authors.join("、")};动 ${e.spot}):${parts.join(";")}`;
|
||||
}
|
||||
|
||||
/**
|
||||
* 第 round 轮诊断的结果:每个实验做了什么、实测、用时、结局,每条诊断解释确认或否定,程序推出的方向;没诊断时写原因。
|
||||
* detail 为 true 时每条解释带正文(公共材料),否则只列确认与否定(信念本的诊断历史)。
|
||||
*/
|
||||
export function diagnosisLines(board: Board, study: StudyFacts, round: number, detail: boolean): string[] {
|
||||
const s = board.diagnosisSettlement(round);
|
||||
if (s === undefined) return [`本轮没有诊断:${noDiagnosisReason(board, study, round)}`];
|
||||
const process = board.runs().find((r) => r.plan.run === s.run)!.result!.process;
|
||||
const lines = [
|
||||
`诊断运行 ${s.run}:在基线 ${board.baselineOf(round)}(最终指标 ${score(s.input.baseline)})上,一次运行做了 ${s.judged.experiments.length} 个实验,只扣 1 次。` +
|
||||
"诊断实验的答案不采纳为基线,只用来看这个方向还剩多少空间。",
|
||||
resourcesText(process, board.publicRunOf(ORIGINAL_BASELINE)!.result!.process, study.wallSeconds),
|
||||
...["", "诊断实验:", ...board.diagnosisExperiments(round).map((e) => experimentLine(board, study, round, s, e)), "", "诊断解释:"],
|
||||
];
|
||||
for (const x of board.diagnosisExplanations(round)) {
|
||||
const verdicts = s.output.judgments.find((j) => j.id === x.id)!.verdicts;
|
||||
const of = (v: string) => verdicts.filter((y) => y.verdict === v).map((y) => y.experiment).join("、") || "无";
|
||||
lines.push(`- ${x.id}(作者 ${x.author}):${verdicts.length === 0 ? "没有有效实验可判" : `确认 ${of("confirmed")};否定 ${of("denied")}`}`);
|
||||
if (detail) lines.push(...x.body.split("\n").map((l) => ` ${l}`));
|
||||
}
|
||||
const d = s.output.direction;
|
||||
const gain = d.gains.find((g) => g.experiment === d.experiment)!;
|
||||
const way = `程序按诊断推出的方向:${d.experiment}(${gain.measured ? "实测" : "按更新后的权重估计"} ${signed(d.gain)})。这只是按公式推出来的参考,你可以不跟。`;
|
||||
return [...lines, "", d.found ? way : "诊断没有指出能涨分的方向:所有实验的收益都不为正。"];
|
||||
}
|
||||
|
||||
/** 信念本的"诊断历史"(方法.md 6.10 第 3 条):每个诊断这一段已走完的轮次一段。 */
|
||||
export function diagnosisHistory(board: Board, study: StudyFacts, latestRound: number): string[] {
|
||||
const rounds = Array.from({ length: latestRound }, (_, i) => i + 1).filter((r) => board.diagnosisDone(r));
|
||||
return rounds.length === 0 ? ["- 还没有"] : rounds.flatMap((r) => [`- 第 ${r} 轮:`, ...diagnosisLines(board, study, r, false).map((l) => (l === "" ? l : ` ${l}`))]);
|
||||
}
|
||||
@@ -7,7 +7,7 @@ export { buildMaterial } from "./build.ts";
|
||||
export type { CodeChanges, PromptFile, Prompts, StudyFacts, TaskRequest } from "./inputs.ts";
|
||||
export { queryMaterial, readPrompts } from "./prompts.ts";
|
||||
// flow 推进时读账本也用这一份(本批基线、基线分数、留池候选、各候选合并了哪些原提案),不另写一份。
|
||||
export { Board, validScore } from "./read.ts";
|
||||
export { Board, validScore, type MergedExperiment } from "./read.ts";
|
||||
export {
|
||||
callId,
|
||||
candidateId,
|
||||
@@ -17,6 +17,7 @@ export {
|
||||
clauseKey,
|
||||
critiqueId,
|
||||
DECISION_KINDS,
|
||||
DIAGNOSIS_NOT_RUN, diagnosisExplanationId, diagnosisId, diagnosisKey, diagnosisNotRunEvent,
|
||||
entryId,
|
||||
experimenterId,
|
||||
verifierId,
|
||||
@@ -33,6 +34,7 @@ export {
|
||||
type DecisionKind,
|
||||
type DecisionShapes,
|
||||
type DedupOutput,
|
||||
type DiagnosisDedupOutput, type DiagnosisLocationTable, type DiagnosisSettlement,
|
||||
type FrozenProtocol,
|
||||
type LocationTable,
|
||||
type Settlement,
|
||||
|
||||
@@ -17,8 +17,10 @@ export type StudyFacts = {
|
||||
lambda: number;
|
||||
/** 正式运行总次数。 */
|
||||
totalRuns: number;
|
||||
/** 每人每轮提案上限 M。 */
|
||||
/** 每人每轮提案上限 M,诊断解释的上限也是它。 */
|
||||
proposalLimit: number;
|
||||
/** 每人每轮诊断实验上限 A;诊断关掉时为 null。 */
|
||||
diagnosisExperimentLimit: number | null;
|
||||
factorLimit: number;
|
||||
/** 最终指标和常规辅助指标的定义(tasks 的 metricDefinitions)。 */
|
||||
metrics: MetricDefinition[];
|
||||
@@ -50,15 +52,15 @@ export type Prompts = { roles: Record<Role, PromptFile>; tasks: Record<Submissio
|
||||
|
||||
/**
|
||||
* 生成一份任务材料要的全部输入。全部必填,用不到的写 null:
|
||||
* - changes:核验是起点到候选代码,修复是起点到上一次提交的代码,消融核验是完整代码到这一格的代码;其余为 null;
|
||||
* - artifacts:解释任务要的本轮各次运行的产物(运行编号 → 文件名 → 文字);其余为 null;
|
||||
* - changes:核验是起点到候选代码,修复是起点到上一次提交的代码,消融核验是完整代码到这一格的代码;诊断核验是当前基线到诊断程序,诊断修复是当前基线到上一次提交的诊断程序;其余为 null;
|
||||
* - artifacts:解释任务要的本轮各次运行的产物、诊 1 要的本批基线那次运行的产物(运行编号 → 文件名 → 文字);其余为 null;
|
||||
* - codeRoot:能读的代码版本取出到 `<codeRoot>/<版本号>`,flow 照材料里的目录取出。
|
||||
*/
|
||||
export type TaskRequest = {
|
||||
kind: SubmissionKind;
|
||||
actor: string;
|
||||
stageId: string;
|
||||
/** 一个任务一个对象:候选、原提案(挑刺)或消融格;入口、讨论、志愿、提案、解释、收尾为 null。 */
|
||||
/** 一个任务一个对象:候选、原提案(挑刺)、消融格或本轮诊断(diagnosisId);入口、讨论、志愿、提案、诊断解释与预测、解释、收尾为 null。 */
|
||||
object: string | null;
|
||||
ledger: Ledger;
|
||||
study: StudyFacts;
|
||||
|
||||
+49
-3
@@ -1,11 +1,13 @@
|
||||
import type { Content, LedgerRecord, StoredSubmission } from "../ledger/index.ts";
|
||||
import { compareIds, type RunState } from "../rules/index.ts";
|
||||
import type { JudgedResult, Range, Stage, SubmissionKind } from "../vocab/index.ts";
|
||||
import type { HypothesisMetric, JudgedResult, Range, Stage, SubmissionKind } from "../vocab/index.ts";
|
||||
import {
|
||||
cellId,
|
||||
diagnosisExplanationId,
|
||||
ORIGINAL_BASELINE,
|
||||
type DecisionKind,
|
||||
type DecisionShapes,
|
||||
type DiagnosisSettlement,
|
||||
type FrozenProtocol,
|
||||
type LocationTable,
|
||||
type Settlement,
|
||||
@@ -22,6 +24,9 @@ export type Run = {
|
||||
result: Content<"runResult"> | null;
|
||||
};
|
||||
|
||||
/** 本轮合并后的一个诊断实验:编号、全部作者、正文(取作者里编号最小的那一份)、登记的指标(编号已是全名"实验编号.短名")。 */
|
||||
export type MergedExperiment = { id: string; authors: string[]; name: string; spot: string; body: string; metrics: HypothesisMetric[] };
|
||||
|
||||
type StageInfo = { stage: Stage; round: number; roster: string[]; closed: boolean };
|
||||
|
||||
export class Board {
|
||||
@@ -116,12 +121,14 @@ export class Board {
|
||||
|
||||
/**
|
||||
* 运行什么时候公开(任务书 09 设计稿):基线运行有结果即公开;第 r 轮正向运行在第 r 轮的采纳决定写入后
|
||||
* 整批公开(方法.md 4.3 ⑪);第 r 轮消融运行在第 r 轮结算写入后公开(4.3 ⑰ ⑱)。
|
||||
* 整批公开(方法.md 4.3 ⑪);第 r 轮消融运行在第 r 轮结算写入后公开(4.3 ⑰ ⑱);
|
||||
* 第 r 轮诊断运行在第 r 轮诊断结算写入后公开(4.3 诊 4)。
|
||||
*/
|
||||
runPublic(run: Run): boolean {
|
||||
if (run.result === null) return false;
|
||||
if (run.plan.purpose === "baseline") return true;
|
||||
if (run.plan.purpose === "forward") return this.decision("adopt", run.plan.round) !== undefined;
|
||||
if (run.plan.purpose === "diagnosis") return this.diagnosisSettlement(run.plan.round) !== undefined;
|
||||
return this.settlement(run.plan.round) !== undefined;
|
||||
}
|
||||
|
||||
@@ -140,7 +147,12 @@ export class Board {
|
||||
|
||||
/** 每次运行在账上的状态,算余额用(方法.md 第七部分)。 */
|
||||
runStates(): RunState[] {
|
||||
return this.runs().map((r) => (r.result?.process.ending === "notStarted" ? "notStarted" : r.started ? "started" : "reserved"));
|
||||
return this.runs().map(stateOf);
|
||||
}
|
||||
|
||||
/** 第 round 轮开始前各次运行在账上的状态(算诊断开始时的余额)。 */
|
||||
runStatesBefore(round: number): RunState[] {
|
||||
return this.runs().filter((r) => r.plan.round < round).map(stateOf);
|
||||
}
|
||||
|
||||
/** 某个候选最新的实现或修复提交(已关阶段)。 */
|
||||
@@ -169,6 +181,38 @@ export class Board {
|
||||
return this.#of("settlement").map((r) => ({ round: r.content.round, ...(r.content.result as unknown as Settlement) }));
|
||||
}
|
||||
|
||||
/** 第 round 轮的诊断结算(还没写入为 undefined)。 */
|
||||
diagnosisSettlement(round: number): DiagnosisSettlement | undefined {
|
||||
return this.diagnosisSettlements().find((s) => s.round === round);
|
||||
}
|
||||
|
||||
diagnosisSettlements(): Array<{ round: number } & DiagnosisSettlement> {
|
||||
return this.#of("diagnosisSettlement").map((r) => ({ round: r.content.round, ...(r.content.result as unknown as DiagnosisSettlement) }));
|
||||
}
|
||||
|
||||
/** 第 round 轮诊断这一段走完了没有:提案阶段开了,就是结算写了或本轮不诊断。 */
|
||||
diagnosisDone(round: number): boolean {
|
||||
return [...this.stages.values()].some((s) => s.stage === "proposal" && s.round === round);
|
||||
}
|
||||
|
||||
/** 第 round 轮已公开的诊断解释,按作者、条序排。 */
|
||||
diagnosisExplanations(round: number): Array<{ id: string; author: string; body: string }> {
|
||||
return this.published("diagnosisProposal")
|
||||
.filter((s) => s.round === round)
|
||||
.sort((a, b) => compareIds(a.author, b.author))
|
||||
.flatMap((s) => s.content.explanations.map((x, i) => ({ id: diagnosisExplanationId(round, s.author, i + 1), author: s.author, body: x.body })));
|
||||
}
|
||||
|
||||
/** 第 round 轮合并后的诊断实验:判重合并的决定指明合成自哪几个作者的哪几个实验,正文取第一个。 */
|
||||
diagnosisExperiments(round: number): MergedExperiment[] {
|
||||
const proposals = this.published("diagnosisProposal").filter((s) => s.round === round);
|
||||
const experimentOf = (src: { author: string; index: number }) => proposals.find((s) => s.author === src.author)!.content.experiments[src.index]!;
|
||||
return this.decision("diagnosisDedup", round)!.output.experiments.map(({ experiment, sources }) => {
|
||||
const { name, spot, body, metrics } = experimentOf(sources[0]!);
|
||||
return { id: experiment, authors: [...new Set(sources.map((x) => x.author))], name, spot, body, metrics: metrics.map((m) => ({ ...m, id: `${experiment}.${m.id}` })) };
|
||||
});
|
||||
}
|
||||
|
||||
/** 本人最新的私人笔记(方法.md 第三部分"私人笔记"),没写过为 null。 */
|
||||
note(author: string): string | null {
|
||||
return this.#of("note").filter((r) => r.content.author === author).at(-1)?.content.body ?? null;
|
||||
@@ -249,6 +293,8 @@ export class Board {
|
||||
}
|
||||
}
|
||||
|
||||
const stateOf = (r: Run): RunState => (r.result?.process.ending === "notStarted" ? "notStarted" : r.started ? "started" : "reserved");
|
||||
|
||||
/** 判分结果里的有效分数:算数且测到了分数才有,否则 null。 */
|
||||
export function validScore(judged: JudgedResult | null): number | null {
|
||||
return judged !== null && judged.verdict === "valid" && judged.score.status === "measured" ? judged.score.value : null;
|
||||
|
||||
+38
-12
@@ -1,8 +1,11 @@
|
||||
import { compareIds } from "../rules/index.ts";
|
||||
import { ANSWER_PATH_KEY, SAFEGUARD_TRIGGERS_KEY, type ChecklistOf } from "../vocab/index.ts";
|
||||
import {
|
||||
artifactSection,
|
||||
baselineRunSection,
|
||||
beliefSection,
|
||||
codeSection,
|
||||
diagnosisSection,
|
||||
Draft,
|
||||
entryVersions,
|
||||
factorLines,
|
||||
@@ -33,6 +36,7 @@ function frame(d: Draft, own: Own, middle: () => void): void {
|
||||
researcherTaskSection(d);
|
||||
codeSection(d);
|
||||
if (d.round >= 1) {
|
||||
diagnosisSection(d);
|
||||
publicRecords(d);
|
||||
beliefSection(d);
|
||||
}
|
||||
@@ -126,16 +130,42 @@ function concernLines(d: Draft, round: number): string[] {
|
||||
return lines.length === 0 ? ["(没有)"] : lines;
|
||||
}
|
||||
|
||||
/** 上一轮全部研究者的收尾(提案、诊 1 共用)。 */
|
||||
function previousWrapSection(d: Draft): void {
|
||||
const prev = d.round - 1;
|
||||
const wraps = d.board.published("wrapUp").filter((w) => w.round === prev);
|
||||
d.section("上一轮的收尾", prev < 1 ? ["(这是第 1 轮,没有上一轮)"] : wraps.flatMap((w) => [`### ${w.author} 的收尾`, "", w.content.body, ""]));
|
||||
}
|
||||
|
||||
/** 诊 1 诊断解释与诊断实验:当前基线的代码,产出它的那次运行的答案、分数、指标,信念本,上一轮收尾;第 1 轮的第零轮全部记录在研究记录里。 */
|
||||
export function diagnosisProposal(d: Draft): ChecklistOf<"diagnosisProposal"> {
|
||||
frame(d, [], () => {
|
||||
baselineRunSection(d);
|
||||
artifactSection(d, "本批基线那次运行的答案");
|
||||
previousWrapSection(d);
|
||||
});
|
||||
return { kind: "diagnosisProposal", maxExplanations: d.req.study.proposalLimit, maxExperiments: d.req.study.diagnosisExperimentLimit!, takenMetricIds: factorRules(d).takenMetricIds };
|
||||
}
|
||||
|
||||
/** 诊 2 诊断预测:合并后的诊断实验清单(标明作者)、全部诊断解释;自己的解释另列在"你自己的"里。别人在这一步交的数一概不读(只经 Board.mine 读本人的)。 */
|
||||
export function diagnosisPrediction(d: Draft): ChecklistOf<"diagnosisPrediction"> {
|
||||
const all = d.board.diagnosisExplanations(d.round);
|
||||
const mine = all.filter((x) => x.author === d.req.actor);
|
||||
const experiments = d.board.diagnosisExperiments(d.round);
|
||||
const text = (x: { id: string; author: string; body: string }) => [`#### ${x.id}(作者 ${x.author})`, "", x.body, ""];
|
||||
frame(d, [{ heading: "你的诊断解释", lines: ["下面每一条都要对每个合并后的诊断实验各给三个数。", "", ...mine.flatMap(text)] }], () => {
|
||||
baselineRunSection(d);
|
||||
const merged = experiments.flatMap((e) => [`#### ${e.id} ${e.name}(作者 ${e.authors.join("、")})`, "", `动哪一处:${e.spot}`, "", e.body, ""]);
|
||||
d.section("本任务要处理的东西", ["### 合并后的诊断实验", "", ...merged, "### 全部诊断解释(含你自己的)", "", ...all.flatMap(text)]);
|
||||
});
|
||||
return { kind: "diagnosisPrediction", explanations: mine.map((x) => x.id), experiments: experiments.map((e) => e.id) };
|
||||
}
|
||||
|
||||
/** ② 提案:公共材料,加上一轮全部收尾、上一轮实验员的提醒、可以写进"延续"的因子。 */
|
||||
export function proposal(d: Draft): ChecklistOf<"proposal"> {
|
||||
frame(d, [], () => {
|
||||
const prev = d.round - 1;
|
||||
const wraps = d.board.published("wrapUp").filter((w) => w.round === prev);
|
||||
d.section(
|
||||
"上一轮的收尾",
|
||||
prev < 1 ? ["(这是第 1 轮,没有上一轮)"] : wraps.flatMap((w) => [`### ${w.author} 的收尾`, "", w.content.body, ""]),
|
||||
);
|
||||
d.section("上一轮实验员实现时的提醒", prev < 1 ? ["(这是第 1 轮,没有上一轮)"] : concernLines(d, prev));
|
||||
previousWrapSection(d);
|
||||
d.section("上一轮实验员实现时的提醒", d.round < 2 ? ["(这是第 1 轮,没有上一轮)"] : concernLines(d, d.round - 1));
|
||||
const lines = d.board.protocols().flatMap((p) =>
|
||||
p.protocol.factors.map((f, i) => `- ${factorId(p.candidate, factorKey(i))} ${f.name}(关键设定:${f.keySettings};候选 ${p.candidate},基线 ${p.protocol.baseline}${f.continues === null ? "" : `;它自己延续 ${f.continues}`})`),
|
||||
);
|
||||
@@ -258,11 +288,7 @@ export function explanation(d: Draft): ChecklistOf<"explanation"> {
|
||||
"",
|
||||
...factorLines(p),
|
||||
]);
|
||||
const artifacts = d.req.artifacts!;
|
||||
const listed = Object.keys(artifacts)
|
||||
.sort(compareIds)
|
||||
.flatMap((run) => Object.entries(artifacts[run]!).map(([file, text]) => `- ${d.record(`${run}/${file}`, `运行 ${run} 的产物 ${file}`, text)}`));
|
||||
d.section("本轮运行的产物", ["用 research_query 按编号查全文:", ...(listed.length === 0 ? ["(没有)"] : listed)]);
|
||||
artifactSection(d, "本轮运行的产物");
|
||||
});
|
||||
return { kind: "explanation", cells: p.factors.map((_, i) => factorKey(i)), maxExplanations: d.req.study.proposalLimit };
|
||||
}
|
||||
|
||||
+41
-2
@@ -5,13 +5,17 @@ import type {
|
||||
BetResult,
|
||||
CellDecision,
|
||||
CellRuleInput,
|
||||
DiagnosisDecision,
|
||||
DiagnosisRuleInput,
|
||||
EntryAssignment,
|
||||
SettleDiagnosisInput,
|
||||
SettleInput,
|
||||
adopt,
|
||||
settleAblation,
|
||||
settleDiagnosis,
|
||||
} from "../rules/index.ts";
|
||||
import type { Content } from "../ledger/index.ts";
|
||||
import type { Json, Role, Submission, SubmissionKind } from "../vocab/index.ts";
|
||||
import type { Json, Measured, Role, Submission, SubmissionKind, Verdict } from "../vocab/index.ts";
|
||||
|
||||
// 账本里只存成 Json 的几样内容长什么样、编号怎么起(用户 2026-10-03 决定 2):materials 读它们,flow 照着写。
|
||||
// 决定和结算的输出直接用 rules 的返回类型;ledger 不认识这些类型,只存 Json。
|
||||
@@ -51,6 +55,12 @@ export function splitCell(cell: string): { candidate: string; key: string } {
|
||||
}
|
||||
/** 解释:第 round 轮作者交的第 i 条(从 1 数)。结算里判确认、否定用它。 */
|
||||
export const explanationId = (round: number, author: string, i: number): string => `r${round}-${author}-x${i}`;
|
||||
/** 本轮诊断:诊断程序的冻结代码、诊断运行、诊断实现与核验的对象都是它。 */
|
||||
export const diagnosisId = (round: number): string => `r${round}-diagnosis`;
|
||||
/** 诊断实验在本轮合并后清单里的编号:第 i 个(从 0 数)是 d(i+1);它登记的指标全名是"实验编号.短名"。 */
|
||||
export const diagnosisKey = (i: number): string => `d${i + 1}`;
|
||||
/** 诊断解释:第 round 轮作者交的第 i 条(从 1 数)。 */
|
||||
export const diagnosisExplanationId = (round: number, author: string, i: number): string => `r${round}-${author}-dx${i}`;
|
||||
/** 模型调用:同一个任务(阶段、谁、哪种提交、对象)第 attempt 次发起。 */
|
||||
export const callId = (stageId: string, actor: string, kind: string, object: string | null, attempt: number): string =>
|
||||
`${stageId}.${actor}.${kind}.${object ?? "none"}.${attempt}`;
|
||||
@@ -104,11 +114,19 @@ export function freezeProtocol(body: ProtocolBody, candidate: string, round: num
|
||||
|
||||
/** 候选代码的位置表,就是实验员交的那三项;基线和消融格的代码记 null(消融格用完整代码的位置表)。 */
|
||||
export type LocationTable = Pick<Submission<"implementation">, "factorLocations" | "safeguardLocations" | "unassigned">;
|
||||
/** 诊断程序的位置表:每个选中的实验一项(诊断实现交的 locations)。 */
|
||||
export type DiagnosisLocationTable = Submission<"diagnosisImplementation">["locations"];
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// 决定(账本记录 decision:decision 是种类,input、output 如下)
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
/**
|
||||
* 诊断实验判重合并的输出(方法.md 4.3 诊 2):合并后的每个实验由哪几个作者的哪几个实验(index 是它在作者那份诊断提案里的位置,从 0 数)合成;
|
||||
* sources 按作者编号排,第一个就是"作者里编号最小的那一份",合并后的正文取它的。
|
||||
*/
|
||||
export type DiagnosisDedupOutput = { experiments: Array<{ experiment: string; sources: Array<{ author: string; index: number }> }> };
|
||||
|
||||
/** 判重合并的输出(方法.md 4.3 ③):本轮新候选各合并了哪几条提案;留池候选各并进了哪几条新提案(可以没有)。 */
|
||||
export type DedupOutput = {
|
||||
fresh: Array<{ candidate: string; proposals: string[] }>;
|
||||
@@ -127,9 +145,11 @@ export type DecisionShapes = {
|
||||
adopt: { input: Parameters<typeof adopt>[0]; output: ReturnType<typeof adopt> };
|
||||
cells: { input: CellRuleInput; output: CellDecision };
|
||||
bet: { input: AblationModelInput & { cells: SettleInput["cells"] }; output: BetResult };
|
||||
diagnosisDedup: { output: DiagnosisDedupOutput };
|
||||
selectDiagnosis: { input: DiagnosisRuleInput; output: DiagnosisDecision };
|
||||
};
|
||||
export type DecisionKind = keyof DecisionShapes;
|
||||
export const DECISION_KINDS: readonly DecisionKind[] = ["entries", "dedup", "batch", "adopt", "cells", "bet"];
|
||||
export const DECISION_KINDS: readonly DecisionKind[] = ["entries", "dedup", "batch", "adopt", "cells", "bet", "diagnosisDedup", "selectDiagnosis"];
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// 结算(账本记录 settlement.result,方法.md 4.3 ⑰)
|
||||
@@ -142,6 +162,25 @@ export type Settlement = {
|
||||
output: ReturnType<typeof settleAblation>;
|
||||
};
|
||||
|
||||
/**
|
||||
* 一次诊断的结算(账本记录 diagnosisSettlement.result,方法.md 6.12)。judged 是每个跑过的实验单独判分的结果,与 tasks 的
|
||||
* DiagnosisJudged 同形(materials 不引用 tasks);诊断运行记录的 judged 为 null,逐个实验的结果只在这里。
|
||||
* dropped 是核验不通过、修复用完而没跑的实验和原因;input、output 是 settleDiagnosis 的输入输出,没做的实验在里面记 untested。
|
||||
*/
|
||||
export type DiagnosisSettlement = {
|
||||
run: string;
|
||||
judged: {
|
||||
experiments: Array<{ id: string; ending: "finished" | "crashed" | "timeout" | "notRun"; wallSeconds: Measured; verdict: Verdict; verdictReason: string; score: Measured; metrics: Record<string, Measured>; extra: Json }>;
|
||||
};
|
||||
dropped: Array<{ experiment: string; reason: string }>;
|
||||
input: SettleDiagnosisInput;
|
||||
output: ReturnType<typeof settleDiagnosis>;
|
||||
};
|
||||
|
||||
/** 事件(账本记录 event):选中的诊断实验核验都没通过、修复用完,一个都不剩,不跑、不扣次(方法.md 4.3 诊 4)。objects 只有本轮诊断的编号。 */
|
||||
export const DIAGNOSIS_NOT_RUN = "diagnosisNotRun";
|
||||
export const diagnosisNotRunEvent = (round: number, reason: string): Content<"event"> => ({ event: DIAGNOSIS_NOT_RUN, reason, objects: [diagnosisId(round)] });
|
||||
|
||||
/**
|
||||
* 事件(账本记录 event):一格消融核验不通过、重做次数用完,这一格不跑、记"证据不足"(方法.md 4.3 ⑯、6.6)。
|
||||
* 这一格没有运行,信念本从这条事件读原因。objects 只有这一格的编号。
|
||||
|
||||
+20
-15
@@ -1,9 +1,17 @@
|
||||
import type { AblationCell } from "./ablation.ts";
|
||||
import { compareIds, improvement, NET_TOLERANCE } from "./basics.ts";
|
||||
import { ablationCellLimit } from "./budget.ts";
|
||||
import { density } from "./distribution.ts";
|
||||
import { seededSample } from "./random.ts";
|
||||
import { ablationModel, evaluateCombos, type AblationModelInput, type ComboParams, type ComboValue } from "./value.ts";
|
||||
import {
|
||||
ablationModel,
|
||||
ablationView,
|
||||
evaluateCombos,
|
||||
itemGain,
|
||||
updatedWeights,
|
||||
type AblationModelInput,
|
||||
type ComboParams,
|
||||
type ComboValue,
|
||||
} from "./value.ts";
|
||||
|
||||
// 方法.md 6.7"选格"、第十一部分的四条选格规则,以及 6.8 押的因子。
|
||||
|
||||
@@ -121,10 +129,15 @@ function netOf(combos: readonly ComboValue[], key: string): number {
|
||||
* "含这一格的净值最大组合"直接按这条挑(只是记录,不要求净值 > 1e-12)。
|
||||
*/
|
||||
export function topCombo<C extends { factors: readonly string[]; net: number }>(combos: readonly C[]): C | null {
|
||||
return topByNet(combos, (c) => c.factors);
|
||||
}
|
||||
|
||||
/** topCombo 的比较规则,组合里的项(因子或实验)怎么取由 items 给;选格和选诊断实验共用(6.7、6.12)。 */
|
||||
export function topByNet<C extends { net: number }>(combos: readonly C[], items: (c: C) => readonly string[]): C | null {
|
||||
if (combos.length === 0) return null;
|
||||
const top = Math.max(...combos.map((c) => c.net));
|
||||
const near = combos.filter((c) => c.net >= top - NET_TOLERANCE);
|
||||
return near.reduce((best, c) => (compareCombos(c.factors, best.factors) < 0 ? c : best));
|
||||
return near.reduce((best, c) => (compareCombos(items(c), items(best)) < 0 ? c : best));
|
||||
}
|
||||
|
||||
/** 格数少的在前;格数相同时按因子编号逐个比。 */
|
||||
@@ -207,19 +220,11 @@ export function chooseBet(input: AblationModelInput & { cells: readonly Ablation
|
||||
const o = input.cells.find((c) => c.factor === f)?.outcome;
|
||||
return o?.kind === "valid" ? o.score : null;
|
||||
});
|
||||
const raw = model.weights.map((w, j) =>
|
||||
scores.reduce<number>((v, x, c) => (x === null ? v : v * density(model.rawKnots[c]![j]!, x)), w),
|
||||
);
|
||||
const sum = raw.reduce((t, v) => t + v, 0);
|
||||
const weightsKept = sum === 0;
|
||||
const weights = weightsKept ? [...model.weights] : raw.map((v) => v / sum);
|
||||
const view = ablationView(model);
|
||||
const { weights, kept: weightsKept } = updatedWeights(view, scores);
|
||||
const gains = model.factors.map((factor, c) => {
|
||||
const x = scores[c];
|
||||
const gain =
|
||||
x !== null && x !== undefined
|
||||
? model.space * (improvement(x, input.full, input.direction) / model.gain)
|
||||
: model.space * model.expected[c]!.reduce((t, e, j) => t + weights[j]! * e, 0);
|
||||
return { factor, gain, measured: x !== null };
|
||||
const x = scores[c] ?? null;
|
||||
return { factor, gain: itemGain(view, c, weights, x), measured: x !== null };
|
||||
});
|
||||
let best = 0;
|
||||
for (let c = 1; c < gains.length; c++) if (gains[c]!.gain > gains[best]!.gain) best = c;
|
||||
|
||||
@@ -0,0 +1,304 @@
|
||||
import type { Direction, Measured, Range } from "../vocab/index.ts";
|
||||
import { judgeExplanations, type AblationExplanation, type CellOutcome, type ExplanationVerdict } from "./ablation.ts";
|
||||
import { combinations, compareIds, improvement, NET_TOLERANCE } from "./basics.ts";
|
||||
import { bitWidth, disagreement, surprise, worldBits } from "./bits.ts";
|
||||
import { topByNet } from "./cells.ts";
|
||||
import { ablationCellMixture, knotsMean, mapKnots, mixtureRange } from "./distribution.ts";
|
||||
import {
|
||||
itemGain,
|
||||
OPEN_RESERVE,
|
||||
runCost,
|
||||
updatedWeights,
|
||||
usePotential,
|
||||
valueOf,
|
||||
type ComboParams,
|
||||
type ValueModel,
|
||||
} from "./value.ts";
|
||||
|
||||
// 方法.md 6.12 诊断:成分模型、方向收益、信息价值、价值减成本、选实验、结算。
|
||||
// 成分、权重更新、精确积分、部分格网格、价值减成本的公式都在 value.ts,这里只给出诊断的"实测 → 收益"换算:
|
||||
// 收益 g(x) = 改进量 x − b(越小越好的任务按 6.1 换成 b − x),另有收益为 0 的"都不走"。
|
||||
|
||||
/** 一条诊断解释:作者,以及对每个合并后的诊断实验给的三个数(实验做完后的分数,原值)。 */
|
||||
export interface DiagnosisExplanation {
|
||||
id: string;
|
||||
author: string;
|
||||
experiments: ReadonlyArray<{ experiment: string; range: Range }>;
|
||||
}
|
||||
|
||||
export interface DiagnosisModelInput {
|
||||
/** 本批基线 b。 */
|
||||
baseline: number;
|
||||
direction: Direction;
|
||||
lambda: number;
|
||||
/** 合并后的全部诊断实验编号(包括没选进组合的)。 */
|
||||
experiments: readonly string[];
|
||||
/** 每条解释对每个实验都给了三个数。 */
|
||||
explanations: readonly DiagnosisExplanation[];
|
||||
}
|
||||
|
||||
export interface DiagnosisModel extends ValueModel {
|
||||
/** 实验编号,按编号排;ValueModel 里"项的下标"按这个次序。 */
|
||||
experiments: string[];
|
||||
/** 成分:各条解释的编号(按编号排),最后是开放预留。 */
|
||||
components: string[];
|
||||
}
|
||||
|
||||
/**
|
||||
* 成分模型(6.12"成分与权重"):每条诊断解释一个成分,开放预留一个成分;每个成分对每个实验有一个分布,
|
||||
* 解释用它自己的三个数补尾巴,开放预留用这个实验自己的均匀分布(6.3"诊断",和消融一格同一个合并函数)。
|
||||
*/
|
||||
export function diagnosisModel(input: DiagnosisModelInput): DiagnosisModel {
|
||||
const experiments = [...input.experiments].sort(compareIds);
|
||||
const explanations = [...input.explanations].sort((a, b) => compareIds(a.id, b.id));
|
||||
const mixtures = experiments.map((d) =>
|
||||
ablationCellMixture(
|
||||
explanations.map((e) => ({ id: e.id, author: e.author, range: e.experiments.find((x) => x.experiment === d)!.range })),
|
||||
input.lambda,
|
||||
),
|
||||
);
|
||||
const scale = (x: number): number => improvement(input.baseline, x, input.direction);
|
||||
const rawKnots = mixtures.map((m) => m.map((c) => c.knots));
|
||||
const scaledKnots = rawKnots.map((row) => row.map((k) => mapKnots(k, scale, input.direction === "minimize")));
|
||||
const weights = mixtures[0]!.map((c) => c.weight);
|
||||
const expected = scaledKnots.map((row) => row.map(knotsMean));
|
||||
return {
|
||||
experiments,
|
||||
components: [...explanations.map((e) => e.id), OPEN_RESERVE],
|
||||
weights,
|
||||
rawKnots,
|
||||
scaledKnots,
|
||||
expected,
|
||||
prior: expected.map((row) => row.reduce((s, e, j) => s + weights[j]! * e, 0)),
|
||||
space: 1,
|
||||
floor: 0,
|
||||
scale,
|
||||
};
|
||||
}
|
||||
|
||||
/** 一个诊断实验组合的价值与成本(6.12"价值减成本")。 */
|
||||
export interface DiagnosisCombo {
|
||||
/** 组合里的实验,按编号排。 */
|
||||
experiments: string[];
|
||||
voi: number;
|
||||
/** R̂ = ⌈(余额 − 1) ÷ 每批上限⌉。 */
|
||||
rHat: number;
|
||||
mPrime: number;
|
||||
/** 价值 = u × m′ × VOI。 */
|
||||
value: number;
|
||||
/** 成本:诊断只占一次运行,任何组合都一样。 */
|
||||
cost: number;
|
||||
net: number;
|
||||
}
|
||||
|
||||
export interface DiagnosisParams extends ComboParams {
|
||||
/** 一次诊断最多做几个实验 D(6.11)。 */
|
||||
maxExperiments: number;
|
||||
}
|
||||
|
||||
/**
|
||||
* 列出可选组合并算价值、成本、净值(6.12"选实验"第 1、2 条):余额不到 2 次一个都没有;实验数不超过 D 时只有"全做",
|
||||
* 超过 D 时是所有恰好 D 个实验的组合,按实验编号字典序列出。ĝ₀ = max(0, max_d E(g_d))。
|
||||
*/
|
||||
export function evaluateDiagnosisCombos(
|
||||
model: DiagnosisModel,
|
||||
params: DiagnosisParams,
|
||||
): { g0: number; combos: DiagnosisCombo[] } {
|
||||
const g0 = Math.max(0, ...model.prior);
|
||||
if (params.balance < 2) return { g0, combos: [] };
|
||||
const rHat = Math.ceil((params.balance - 1) / params.batchCap);
|
||||
const mPrime = usePotential(rHat, params.decay);
|
||||
const cost = runCost(g0, params.batchCap, 1);
|
||||
const size = Math.min(model.experiments.length, params.maxExperiments);
|
||||
const combos = combinations(model.experiments.length, size).map((S) => {
|
||||
const voi = valueOf(model, S);
|
||||
const value = params.u * mPrime * voi;
|
||||
return { experiments: S.map((d) => model.experiments[d]!), voi, rHat, mPrime, value, cost, net: value - cost };
|
||||
});
|
||||
return { g0, combos };
|
||||
}
|
||||
|
||||
export interface DiagnosisRuleInput extends DiagnosisModelInput, DiagnosisParams {}
|
||||
|
||||
/**
|
||||
* 没选中的实验为什么没做(6.12"选实验"第 4 条):budget 放不下(已选的组合装满了 D 个);
|
||||
* value 信息价值不足以抵过成本。
|
||||
*/
|
||||
export type SkippedReason = "budget" | "value";
|
||||
|
||||
/** 选诊断实验的结果,含 6.12 要求附上的全部数字;规则名和版本由登记表补上。 */
|
||||
export interface DiagnosisDecision {
|
||||
params: { balance: number; batchCap: number; maxExperiments: number; u: number; decay: number; lambda: number };
|
||||
components: string[];
|
||||
/** 起始权重,与 components 对齐。 */
|
||||
weights: number[];
|
||||
/** 各实验的事前期望收益 E(g_d),按实验编号排。 */
|
||||
priorGains: Array<{ experiment: string; gain: number }>;
|
||||
/** ĝ₀ = max(0, max_d E(g_d))。 */
|
||||
g0: number;
|
||||
/** 可选的全部组合及其价值、成本、净值(余额不到 2 次时为空)。 */
|
||||
combos: DiagnosisCombo[];
|
||||
/** 本轮做不做诊断:选中了净值 > 1e-12 的组合才做。 */
|
||||
diagnose: boolean;
|
||||
/** 选中要做的实验,按编号排;不诊断时为空。 */
|
||||
chosen: string[];
|
||||
/** 选中组合的净值;不诊断时是 0。 */
|
||||
chosenNet: number;
|
||||
/**
|
||||
* 每个合并后的实验:做不做;没做的写原因,并附"含这个实验的组合里净值最大的那个";
|
||||
* 没有任何含它的可选组合(余额不到 2 次,或没被任何组合包含)时 bestCombo 为 null。
|
||||
*/
|
||||
experiments: Array<{
|
||||
experiment: string;
|
||||
done: boolean;
|
||||
reason: SkippedReason | null;
|
||||
bestCombo: { experiments: string[]; net: number } | null;
|
||||
}>;
|
||||
}
|
||||
|
||||
/**
|
||||
* 选诊断实验,规则"信息价值减成本"(6.12"选实验"):取净值最大、且净值 > 1e-12 的组合,净值相差不到 1e-12 的
|
||||
* 视为相同、按实验编号从小到大比(和选格同一条挑法,见 topByNet);都不大于 1e-12 时本轮不诊断。
|
||||
*/
|
||||
export function selectDiagnosis(input: DiagnosisRuleInput): DiagnosisDecision {
|
||||
const model = diagnosisModel(input);
|
||||
const { g0, combos } = evaluateDiagnosisCombos(model, input);
|
||||
const best = topByNet(combos.filter((c) => c.net > NET_TOLERANCE), (c) => c.experiments);
|
||||
const chosen = best === null ? [] : [...best.experiments];
|
||||
return {
|
||||
params: {
|
||||
balance: input.balance,
|
||||
batchCap: input.batchCap,
|
||||
maxExperiments: input.maxExperiments,
|
||||
u: input.u,
|
||||
decay: input.decay,
|
||||
lambda: input.lambda,
|
||||
},
|
||||
components: model.components,
|
||||
weights: model.weights,
|
||||
priorGains: model.experiments.map((experiment, d) => ({ experiment, gain: model.prior[d]! })),
|
||||
g0,
|
||||
combos,
|
||||
diagnose: best !== null,
|
||||
chosen,
|
||||
chosenNet: best === null ? 0 : best.net,
|
||||
experiments: model.experiments.map((experiment) => {
|
||||
const done = chosen.includes(experiment);
|
||||
const top = topByNet(combos.filter((c) => c.experiments.includes(experiment)), (c) => c.experiments);
|
||||
return {
|
||||
experiment,
|
||||
done,
|
||||
reason: done ? null : chosen.length > 0 ? "budget" : "value",
|
||||
bestCombo: done || top === null ? null : { experiments: [...top.experiments], net: top.net },
|
||||
};
|
||||
}),
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* 一个实验的结局(6.12"结算"):valid 有效分数;insufficient 证据不足(跑崩、超时、没有产物、修不了、核验不通过);
|
||||
* untested 没做。和消融一格的三种同形,直接用 ablation.ts 的类型。
|
||||
*/
|
||||
export type ExperimentOutcome = Extract<CellOutcome, { kind: "valid" | "insufficient" | "untested" }>;
|
||||
|
||||
export interface SettleDiagnosisInput {
|
||||
/** 本批基线 b。 */
|
||||
baseline: number;
|
||||
direction: Direction;
|
||||
epsilon: number;
|
||||
lambda: number;
|
||||
/** 合并后的全部诊断实验及结局,没做的写 untested。 */
|
||||
experiments: ReadonlyArray<{ experiment: string; outcome: ExperimentOutcome }>;
|
||||
explanations: readonly DiagnosisExplanation[];
|
||||
}
|
||||
|
||||
/** 诊断推出的方向(6.12"结算")。 */
|
||||
export interface DiagnosisDirection {
|
||||
/** 更新后的成分权重(与模型的成分对齐);无从更新时等于起始权重。 */
|
||||
weights: number[];
|
||||
/** 实测落在所有成分的范围之外、权重之和为 0 时为 true,此时保持起始权重。 */
|
||||
weightsKept: boolean;
|
||||
/** 各实验的收益(按编号排):measured 为 true 的是有效实验实测换算的,其余是按更新后的权重估的。 */
|
||||
gains: Array<{ experiment: string; gain: number; measured: boolean }>;
|
||||
/** 收益最大的实验(严格更大才替换,平局取编号小的)及其收益。 */
|
||||
experiment: string;
|
||||
gain: number;
|
||||
/** 最大收益为正才算指出了方向;否则记"诊断没有指出能涨分的方向"。 */
|
||||
found: boolean;
|
||||
}
|
||||
|
||||
/**
|
||||
* 一次诊断的结算(6.12"结算"、6.9):每个实验的收益、合并范围,有效实验的世界比特、惊讶、分歧量
|
||||
* (格宽 ε/100,从 b 起切);每条诊断解释在每个有效实验上的确认或否定(和消融同一个判法);诊断推出的方向。
|
||||
* 证据不足和没做的实验不判、不记账。
|
||||
*/
|
||||
export function settleDiagnosis(input: SettleDiagnosisInput): {
|
||||
experiments: Array<{
|
||||
experiment: string;
|
||||
outcome: ExperimentOutcome["kind"];
|
||||
gain: number | null;
|
||||
merged: Range;
|
||||
worldBits: number | null;
|
||||
surprise: { cell: number; prior: number; bits: Measured } | null;
|
||||
disagreement: number | null;
|
||||
}>;
|
||||
judgments: Array<{ id: string; verdicts: Array<{ experiment: string; verdict: ExplanationVerdict }> }>;
|
||||
direction: DiagnosisDirection;
|
||||
} {
|
||||
const width = bitWidth(input.epsilon);
|
||||
const experiments = [...input.experiments].sort((a, b) => compareIds(a.experiment, b.experiment));
|
||||
const rows = experiments.map(({ experiment, outcome }) => {
|
||||
const entries = input.explanations.map((e) => ({
|
||||
id: e.id,
|
||||
author: e.author,
|
||||
range: e.experiments.find((x) => x.experiment === experiment)!.range,
|
||||
}));
|
||||
const mixture = ablationCellMixture(entries, input.lambda);
|
||||
const valid = outcome.kind === "valid";
|
||||
return {
|
||||
experiment,
|
||||
outcome: outcome.kind,
|
||||
gain: valid ? improvement(input.baseline, outcome.score, input.direction) : null,
|
||||
merged: mixtureRange(mixture),
|
||||
worldBits: valid ? worldBits(mixture, input.baseline, width) : null,
|
||||
surprise: valid ? surprise(mixture, input.baseline, width, outcome.score) : null,
|
||||
disagreement: valid ? disagreement(entries, input.baseline, width) : null,
|
||||
};
|
||||
});
|
||||
// 确认与否定沿用消融的判法:把"实验"当成"格"交给 judgeExplanations,结果再换回实验的叫法。
|
||||
const asCells: AblationExplanation[] = input.explanations.map((e) => ({
|
||||
id: e.id,
|
||||
author: e.author,
|
||||
cells: e.experiments.map((x) => ({ factor: x.experiment, range: x.range })),
|
||||
}));
|
||||
const judgments = judgeExplanations(asCells, experiments.map((e) => ({ factor: e.experiment, outcome: e.outcome }))).map(
|
||||
(j) => ({ id: j.id, verdicts: j.verdicts.map((v) => ({ experiment: v.factor, verdict: v.verdict })) }),
|
||||
);
|
||||
return {
|
||||
experiments: rows,
|
||||
judgments,
|
||||
direction: diagnosisDirection(
|
||||
diagnosisModel({ ...input, experiments: experiments.map((e) => e.experiment) }),
|
||||
experiments.map((e) => (e.outcome.kind === "valid" ? e.outcome.score : null)),
|
||||
),
|
||||
};
|
||||
}
|
||||
|
||||
/** 诊断推出的方向:按有效实验的实测更新成分权重,有效实验用实测换算收益,其余实验用新权重估,取收益最大的(6.12"结算")。 */
|
||||
function diagnosisDirection(model: DiagnosisModel, scores: ReadonlyArray<number | null>): DiagnosisDirection {
|
||||
const { weights, kept } = updatedWeights(model, scores);
|
||||
const gains = model.experiments.map((experiment, d) => {
|
||||
const x = scores[d] ?? null;
|
||||
return { experiment, gain: itemGain(model, d, weights, x), measured: x !== null };
|
||||
});
|
||||
let best = 0;
|
||||
for (let d = 1; d < gains.length; d++) if (gains[d]!.gain > gains[best]!.gain) best = d;
|
||||
return {
|
||||
weights,
|
||||
weightsKept: kept,
|
||||
gains,
|
||||
experiment: gains[best]!.experiment,
|
||||
gain: gains[best]!.gain,
|
||||
found: gains[best]!.gain > 0,
|
||||
};
|
||||
}
|
||||
@@ -103,6 +103,23 @@ export {
|
||||
worldBits,
|
||||
type Placement,
|
||||
} from "./bits.ts";
|
||||
export {
|
||||
diagnosisModel,
|
||||
evaluateDiagnosisCombos,
|
||||
selectDiagnosis,
|
||||
settleDiagnosis,
|
||||
type DiagnosisCombo,
|
||||
type DiagnosisDecision,
|
||||
type DiagnosisDirection,
|
||||
type DiagnosisExplanation,
|
||||
type DiagnosisModel,
|
||||
type DiagnosisModelInput,
|
||||
type DiagnosisParams,
|
||||
type DiagnosisRuleInput,
|
||||
type ExperimentOutcome,
|
||||
type SettleDiagnosisInput,
|
||||
type SkippedReason,
|
||||
} from "./diagnosis.ts";
|
||||
export { ablationCellLimit, forwardBatchSize, runBalance, type RunState } from "./budget.ts";
|
||||
export { authorVsMerged, trackRecord, type Phase, type Tally } from "./record.ts";
|
||||
export { assignEntries, type EntryAssignment } from "./entries.ts";
|
||||
@@ -110,8 +127,11 @@ export { decisionSeed, seededRandom, seededSample } from "./random.ts";
|
||||
export {
|
||||
BATCH_RULES,
|
||||
CELL_RULES,
|
||||
DIAGNOSIS_RULES,
|
||||
runBatchRule,
|
||||
runCellRule,
|
||||
runDiagnosisRule,
|
||||
type BatchRuleName,
|
||||
type CellRuleName,
|
||||
type DiagnosisRuleName,
|
||||
} from "./registry.ts";
|
||||
|
||||
+7
-5
@@ -2,10 +2,12 @@ import type { Direction, Range } from "../vocab/index.ts";
|
||||
import { compareIds, improvement } from "./basics.ts";
|
||||
import { covers, placement, type Placement } from "./bits.ts";
|
||||
|
||||
// 方法.md 6.10 第 4 条"战绩"、6.9"按人分开、按正向与消融分开"。只用来显示,不改任何权重。
|
||||
// 方法.md 6.10 第 4 条"战绩"、6.9"按人分开、按正向、诊断与消融分开"。只用来显示,不改任何权重。
|
||||
// 一律按分数原值比,不按方向折算。
|
||||
|
||||
export type Phase = "forward" | "ablation";
|
||||
export type Phase = "forward" | "diagnosis" | "ablation";
|
||||
|
||||
const PHASE_ORDER: readonly Phase[] = ["forward", "diagnosis", "ablation"];
|
||||
|
||||
/** 一个人的战绩。 */
|
||||
export interface Tally {
|
||||
@@ -25,8 +27,8 @@ export interface Tally {
|
||||
}
|
||||
|
||||
/**
|
||||
* 战绩:按研究者、按正向与消融分开数(6.9、6.10)。samples 是每人每次的范围(第二次预测或解释的每格)和有效实测。
|
||||
* 结果按研究者编号、再按 forward、ablation 排,只列有样本的组合。
|
||||
* 战绩:按研究者、按正向、诊断与消融分开数(6.9、6.10)。samples 是每人每次的范围(第二次预测、诊断预测或解释的每格)和有效实测。
|
||||
* 结果按研究者编号、再按 forward、diagnosis、ablation 排,只列有样本的组合。
|
||||
*/
|
||||
export function trackRecord(
|
||||
samples: ReadonlyArray<{ researcher: string; phase: Phase; range: Range; value: number }>,
|
||||
@@ -62,7 +64,7 @@ export function trackRecord(
|
||||
t.placements[placement(r, v)] += 1;
|
||||
}
|
||||
return [...groups.values()].sort(
|
||||
(a, b) => compareIds(a.researcher, b.researcher) || (a.phase === b.phase ? 0 : a.phase === "forward" ? -1 : 1),
|
||||
(a, b) => compareIds(a.researcher, b.researcher) || PHASE_ORDER.indexOf(a.phase) - PHASE_ORDER.indexOf(b.phase),
|
||||
);
|
||||
}
|
||||
|
||||
|
||||
@@ -1,4 +1,5 @@
|
||||
import { selectAll, selectByValue, selectNone, selectRandomSameCount, type CellDecision, type CellRuleInput } from "./cells.ts";
|
||||
import { selectDiagnosis, type DiagnosisDecision, type DiagnosisRuleInput } from "./diagnosis.ts";
|
||||
import { selectBestExpected, selectUniformRandom, type BatchDecision, type BatchInput } from "./forward.ts";
|
||||
|
||||
// 规则登记表(结构.md 第五部分"决策规则"、方法.md 第十一部分):名字 → 函数,带版本号。
|
||||
@@ -22,8 +23,14 @@ export const CELL_RULES = {
|
||||
none: { version: 1, run: selectNone },
|
||||
} as const satisfies Record<string, { version: number; run: (input: CellRuleInput) => CellDecision }>;
|
||||
|
||||
export const DIAGNOSIS_RULES = {
|
||||
/** 信息价值减成本(6.12);选诊断实验只有这一条规则,配置里不选。 */
|
||||
valueMinusCost: { version: 1, run: selectDiagnosis },
|
||||
} as const satisfies Record<string, { version: number; run: (input: DiagnosisRuleInput) => DiagnosisDecision }>;
|
||||
|
||||
export type BatchRuleName = keyof typeof BATCH_RULES;
|
||||
export type CellRuleName = keyof typeof CELL_RULES;
|
||||
export type DiagnosisRuleName = keyof typeof DIAGNOSIS_RULES;
|
||||
|
||||
/** 按名字跑一条选批规则,结果带上规则名和版本。 */
|
||||
export function runBatchRule(
|
||||
@@ -42,3 +49,12 @@ export function runCellRule(
|
||||
const entry = CELL_RULES[name];
|
||||
return { rule: { name, version: entry.version }, ...entry.run(input) };
|
||||
}
|
||||
|
||||
/** 按名字跑一条选诊断实验规则,结果带上规则名和版本。 */
|
||||
export function runDiagnosisRule(
|
||||
name: DiagnosisRuleName,
|
||||
input: DiagnosisRuleInput,
|
||||
): DiagnosisDecision & { rule: { name: DiagnosisRuleName; version: number } } {
|
||||
const entry = DIAGNOSIS_RULES[name];
|
||||
return { rule: { name, version: entry.version }, ...entry.run(input) };
|
||||
}
|
||||
|
||||
+139
-31
@@ -9,13 +9,44 @@ import {
|
||||
GRID_SUBDIVISIONS,
|
||||
improvement,
|
||||
} from "./basics.ts";
|
||||
import { ablationCellMixture, below, integrateUpperTail, knotsMean, mapKnots, type Knots } from "./distribution.ts";
|
||||
import {
|
||||
ablationCellMixture,
|
||||
below,
|
||||
density,
|
||||
integrateUpperTail,
|
||||
knotsMean,
|
||||
mapKnots,
|
||||
type Knots,
|
||||
} from "./distribution.ts";
|
||||
|
||||
// 方法.md 6.7:方向收益、成分与权重、信息价值、价值减成本。选格规则和押的因子在 cells.ts。
|
||||
//
|
||||
// 一律先在"撑几成"刻度 s(x) = [f(完整) − x] ÷ G(按方向折算)上算,最后乘剩余空间 h:
|
||||
// 收益_p = h × s_p,h 对所有因子相同且为正,所以信息价值、期望收益、ĝ₀ 都等于 h × (撑几成刻度上的同一个量)。
|
||||
// 例外是部分格网格的切段:在分数原值上取折点、切小段,再把中点换成撑几成(见 measuredGrid)。
|
||||
// 成分、权重更新、精确积分、部分格网格、价值减成本这套引擎消融(6.7)和诊断(6.12)共用,只差"实测 → 收益"的换算:
|
||||
// - 消融:一律先在"撑几成"刻度 s(x) = [f(完整) − x] ÷ G(按方向折算)上算,最后乘剩余空间 h:
|
||||
// 收益_p = h × s_p,h 对所有因子相同且为正,所以信息价值、期望收益、ĝ₀ 都等于 h × (撑几成刻度上的同一个量);
|
||||
// 不设"都不走"的选项。
|
||||
// - 诊断(diagnosis.ts):收益 g(x) = 改进量 x − b,h 取 1,另有收益为 0 的"都不走",即 floor = 0。
|
||||
// 例外是部分格网格的切段:在分数原值上取折点、切小段,再把中点换成收益刻度(见 gridOf)。
|
||||
|
||||
/** 引擎要的成分模型(消融和诊断各自建出来)。下标"项"是因子(消融)或实验(诊断)。 */
|
||||
export interface ValueModel {
|
||||
/** 成分起始权重。 */
|
||||
weights: number[];
|
||||
/** rawKnots[项][成分]:分数原值上的折线。 */
|
||||
rawKnots: Knots[][];
|
||||
/** scaledKnots[项][成分]:收益刻度上的折线。 */
|
||||
scaledKnots: Knots[][];
|
||||
/** expected[项][成分]:这个成分下这一项在收益刻度上的期望。 */
|
||||
expected: number[][];
|
||||
/** prior[项]:按起始权重平均的期望。 */
|
||||
prior: number[];
|
||||
/** 收益 = space × 收益刻度上的值:消融是剩余空间 h,诊断是 1。 */
|
||||
space: number;
|
||||
/** "都不走"的收益:诊断是 0,消融没有这个选项,写 null。 */
|
||||
floor: number | null;
|
||||
/** 分数原值 → 收益刻度。 */
|
||||
scale: (x: number) => number;
|
||||
}
|
||||
|
||||
/** 开放预留成分的名字(成分名单里排在最后)。 */
|
||||
export const OPEN_RESERVE = "openReserve";
|
||||
@@ -70,6 +101,20 @@ function supportScale(full: number, direction: Direction, gain: number): (x: num
|
||||
return (x) => improvement(x, full, direction) / gain;
|
||||
}
|
||||
|
||||
/** 消融模型在引擎眼里的样子:收益刻度是撑几成,没有"都不走"。 */
|
||||
export function ablationView(model: AblationModel): ValueModel {
|
||||
return {
|
||||
weights: model.weights,
|
||||
rawKnots: model.rawKnots,
|
||||
scaledKnots: model.supportKnots,
|
||||
expected: model.expected,
|
||||
prior: model.prior,
|
||||
space: model.space,
|
||||
floor: null,
|
||||
scale: supportScale(model.full, model.direction, model.gain),
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* 成分模型(6.7"成分与权重"):每条解释一个成分,开放预留一个成分;每个成分对每一格有一个分布,
|
||||
* 解释用它自己的三个数补尾巴,开放预留用这一格的均匀分布(6.3"消融")。同一成分内各格互相独立。
|
||||
@@ -144,26 +189,33 @@ export function cellGrid(knots: readonly Knots[], k: number): { mids: number[];
|
||||
}
|
||||
|
||||
/**
|
||||
* 模型里第 c 格的网格(6.7"只测一部分格"),测 k 格时用。在分数原值上取折点、切小段:方法说"每格在自己的开放预留范围内
|
||||
* 模型里第 c 项的网格(6.7"只测一部分格"),测 k 项时用。在分数原值上取折点、切小段:方法说"每格在自己的开放预留范围内
|
||||
* 按折点切段",开放预留和折点都是在分数原值上定的(尾巴按 q1 − (q5 − q1)、q9 + (q9 − q5),预留按 L − W/2、U + W/2
|
||||
* 的浮点原式),去重只合并浮点上完全相等的值。先换到撑几成刻度再去重,相差一个末位的两个原值折点可能换算后变成同一个数,
|
||||
* 段数和 s 就会跟原值上的不一样。切好后只把小段中点换算成撑几成;把握与刻度无关。
|
||||
* 的浮点原式),去重只合并浮点上完全相等的值。先换到收益刻度再去重,相差一个末位的两个原值折点可能换算后变成同一个数,
|
||||
* 段数和 s 就会跟原值上的不一样。切好后只把小段中点换算成收益刻度;把握与刻度无关。
|
||||
*/
|
||||
export function measuredGrid(model: AblationModel, c: number, k: number): { mids: number[]; mass: number[][] } {
|
||||
export function gridOf(model: ValueModel, c: number, k: number): { mids: number[]; mass: number[][] } {
|
||||
const { mids, mass } = cellGrid(model.rawKnots[c]!, k);
|
||||
return { mids: mids.map(supportScale(model.full, model.direction, model.gain)), mass };
|
||||
return { mids: mids.map(model.scale), mass };
|
||||
}
|
||||
|
||||
/** 消融模型里第 c 格的网格,中点换算成撑几成(6.7"只测一部分格")。 */
|
||||
export function measuredGrid(model: AblationModel, c: number, k: number): { mids: number[]; mass: number[][] } {
|
||||
return gridOf(ablationView(model), c, k);
|
||||
}
|
||||
|
||||
/**
|
||||
* 只测 S 里的格时的 E_{S 的结果}[max_p E(撑_p | S 的结果)](6.7"信息价值"),在撑几成刻度上。
|
||||
* 对测到的格的全部小段组合求和:成分 j 的联合把握 = 起始权重 × 各测到格在所选小段的把握;它们的和 marg 是这个组合的把握,
|
||||
* 按它归一就是更新后的权重。网格见 measuredGrid。测到的格取小段中点(已换成撑几成),没测的格用更新后的权重平均各成分的期望,取各因子里最大的。
|
||||
* 只测 S 里的项时的 E_{S 的结果}[max(floor, max_p E(收益_p | S 的结果))](6.7"信息价值"、6.12),在收益刻度上,
|
||||
* 没有"都不走"(floor 为 null)时不和 floor 取大。
|
||||
* 对测到的项的全部小段组合求和:成分 j 的联合把握 = 起始权重 × 各测到项在所选小段的把握;它们的和 marg 是这个组合的把握,
|
||||
* 按它归一就是更新后的权重。网格见 gridOf。测到的项取小段中点(已换成收益刻度),没测的项用更新后的权重平均各成分的期望,取各项里最大的。
|
||||
*/
|
||||
function expectedBestGrid(model: AblationModel, S: readonly number[]): number {
|
||||
function expectedBestGrid(model: ValueModel, S: readonly number[]): number {
|
||||
const k = S.length;
|
||||
const J = model.weights.length;
|
||||
const grids = S.map((c) => measuredGrid(model, c, k));
|
||||
const rest = model.factors.map((_, c) => c).filter((c) => !S.includes(c));
|
||||
const floor = model.floor ?? -Infinity;
|
||||
const grids = S.map((c) => gridOf(model, c, k));
|
||||
const rest = model.prior.map((_, c) => c).filter((c) => !S.includes(c));
|
||||
const partial = Array.from({ length: k + 1 }, () => new Float64Array(J));
|
||||
for (let j = 0; j < J; j++) partial[0]![j] = model.weights[j]!;
|
||||
let total = 0;
|
||||
@@ -173,7 +225,7 @@ function expectedBestGrid(model: AblationModel, S: readonly number[]): number {
|
||||
let marg = 0;
|
||||
for (let j = 0; j < J; j++) marg += cur[j]!;
|
||||
if (marg === 0) return;
|
||||
let best = seen;
|
||||
let best = Math.max(seen, floor);
|
||||
for (const c of rest) {
|
||||
const e = model.expected[c]!;
|
||||
let num = 0;
|
||||
@@ -195,36 +247,80 @@ function expectedBestGrid(model: AblationModel, S: readonly number[]): number {
|
||||
}
|
||||
|
||||
/**
|
||||
* 全部格都测时的 E[max_p 撑_p](6.7"信息价值"):同一成分内"各格撑几成的最大值低于 t"的把握 = 各格把握连乘,
|
||||
* 期望 = 下端 + ∫(1 − 连乘) dt,逐段多项式精确积分;再按起始权重相加。
|
||||
* 全部项都测时的 E[max(floor, max_p 收益_p)](6.7"信息价值"、6.12):同一成分内"各项最大值低于 t"的把握 = 各项把握连乘,
|
||||
* 期望 = 下端 + ∫(1 − 连乘) dt,逐段多项式精确积分;再按起始权重相加。下端:有"都不走"时是 floor(t 低于它时
|
||||
* max(floor, ·) 一定不低于 t,被积函数是 1),没有时是各项折线的最低点。
|
||||
*/
|
||||
function expectedBestAll(model: AblationModel): number {
|
||||
function expectedBestAll(model: ValueModel): number {
|
||||
let v = 0;
|
||||
for (let j = 0; j < model.weights.length; j++) {
|
||||
const lines = model.supportKnots.map((cell) => cell[j]!);
|
||||
const lo = Math.min(...lines.map((k) => k[0]![0]));
|
||||
const lines = model.scaledKnots.map((cell) => cell[j]!);
|
||||
const lo = model.floor ?? Math.min(...lines.map((k) => k[0]![0]));
|
||||
v += model.weights[j]! * (lo + integrateUpperTail(lines.map((knots) => [{ weight: 1, knots }]), lo));
|
||||
}
|
||||
return v;
|
||||
}
|
||||
|
||||
/** max(floor, max_p E(收益_p)):不测时能拿到的最好期望,在收益刻度上。 */
|
||||
function bestPrior(model: ValueModel): number {
|
||||
return Math.max(model.floor ?? -Infinity, ...model.prior);
|
||||
}
|
||||
|
||||
/** 用网格算的信息价值(不论测几项都走网格),S 是项的下标。 */
|
||||
export function valueByGrid(model: ValueModel, S: readonly number[]): number {
|
||||
return model.space * (expectedBestGrid(model, [...S].sort((a, b) => a - b)) - bestPrior(model));
|
||||
}
|
||||
|
||||
/** 全部项都测时的信息价值(一维精确积分)。 */
|
||||
export function valueExact(model: ValueModel): number {
|
||||
return model.space * (expectedBestAll(model) - bestPrior(model));
|
||||
}
|
||||
|
||||
/**
|
||||
* 信息价值 VOI(S) = E_{S 的结果}[max(floor, max_p E(收益_p | S 的结果))] − max(floor, max_p E(收益_p))(6.7、6.12),
|
||||
* S 是项的下标;全部项都测走一维精确积分,只测一部分走网格,空集是 0。不截断。
|
||||
*/
|
||||
export function valueOf(model: ValueModel, S: readonly number[]): number {
|
||||
if (S.length === 0) return 0;
|
||||
return S.length === model.prior.length ? valueExact(model) : valueByGrid(model, S);
|
||||
}
|
||||
|
||||
/** 网格算法算的信息价值,S 是格的下标(不论测几格都走网格;用来和精确算法互相对照)。 */
|
||||
export function informationValueGrid(model: AblationModel, S: readonly number[]): number {
|
||||
return model.space * (expectedBestGrid(model, [...S].sort((a, b) => a - b)) - Math.max(...model.prior));
|
||||
return valueByGrid(ablationView(model), S);
|
||||
}
|
||||
|
||||
/** 全部格都测时的信息价值(一维精确积分)。 */
|
||||
export function informationValueExact(model: AblationModel): number {
|
||||
return model.space * (expectedBestAll(model) - Math.max(...model.prior));
|
||||
return valueExact(ablationView(model));
|
||||
}
|
||||
|
||||
/** 消融的信息价值 VOI(S)(6.7),S 是格的下标。 */
|
||||
export function informationValue(model: AblationModel, S: readonly number[]): number {
|
||||
return valueOf(ablationView(model), S);
|
||||
}
|
||||
|
||||
/**
|
||||
* 信息价值 VOI(S) = E_{S 的结果}[max_p E(收益_p | S 的结果)] − max_p E(收益_p)(6.7),S 是格的下标;
|
||||
* 全部格都测走一维精确积分,只测一部分走网格,空集是 0。不截断。
|
||||
* 成分新权重(6.8、6.12):起始权重 × 各有效项(scores 里不是 null 的)上这个成分在实测处的密度(分数原值上,连乘),
|
||||
* 再归一;实测落在所有成分之外、权重之和为 0 时保持起始权重。scores 与项的下标对齐,没有有效实测的项为 null。
|
||||
*/
|
||||
export function informationValue(model: AblationModel, S: readonly number[]): number {
|
||||
if (S.length === 0) return 0;
|
||||
return S.length === model.factors.length ? informationValueExact(model) : informationValueGrid(model, S);
|
||||
export function updatedWeights(
|
||||
model: ValueModel,
|
||||
scores: ReadonlyArray<number | null>,
|
||||
): { weights: number[]; kept: boolean } {
|
||||
const raw = model.weights.map((w, j) =>
|
||||
scores.reduce<number>((v, x, c) => (x === null ? v : v * density(model.rawKnots[c]![j]!, x)), w),
|
||||
);
|
||||
const sum = raw.reduce((t, v) => t + v, 0);
|
||||
const kept = sum === 0;
|
||||
return { weights: kept ? [...model.weights] : raw.map((v) => v / sum), kept };
|
||||
}
|
||||
|
||||
/** 第 c 项的方向收益(6.8、6.12):有有效实测 x 就用实测换算,否则用 weights 对各成分的期望加权平均。 */
|
||||
export function itemGain(model: ValueModel, c: number, weights: readonly number[], x: number | null): number {
|
||||
return x !== null
|
||||
? model.space * model.scale(x)
|
||||
: model.space * model.expected[c]!.reduce((t, e, j) => t + weights[j]! * e, 0);
|
||||
}
|
||||
|
||||
/** 一个测格组合的价值与成本(6.7"价值减成本")。 */
|
||||
@@ -253,6 +349,19 @@ export interface ComboParams {
|
||||
decay: number;
|
||||
}
|
||||
|
||||
/** m′ = 1 + r + … + r^(R̂−1),R̂ = 0 时是 0(6.7"价值减成本");消融和诊断共用。 */
|
||||
export function usePotential(rHat: number, decay: number): number {
|
||||
let mPrime = 0;
|
||||
for (let j = 0; j < rHat; j++) mPrime += decay ** j;
|
||||
return mPrime;
|
||||
}
|
||||
|
||||
/** 成本 = 2 max(0, ĝ₀) × [(K + n)/(K + n + 1) − K/(K + 1)],n 是这次占用的运行次数(6.7、6.12);消融和诊断共用。 */
|
||||
export function runCost(g0: number, batchCap: number, runs: number): number {
|
||||
const K = batchCap;
|
||||
return 2 * Math.max(0, g0) * ((K + runs) / (K + runs + 1) - K / (K + 1));
|
||||
}
|
||||
|
||||
/**
|
||||
* 列出格数 1..min(格数, 余额 − 1) 的全部非空组合并算价值、成本、净值(6.7"选格"第 1 条)。余额为 0 或 1 时一个都没有。
|
||||
* 组合按格数从少到多、同格数按因子编号字典序列出。ĝ₀ = h × max_p 期望撑几成,原值另给,成本里取 max(0, ĝ₀)。
|
||||
@@ -265,10 +374,9 @@ export function evaluateCombos(model: AblationModel, params: ComboParams): { g0:
|
||||
for (const S of combinations(model.factors.length, size)) {
|
||||
const voi = informationValue(model, S);
|
||||
const rHat = Math.ceil((params.balance - size) / K);
|
||||
let mPrime = 0;
|
||||
for (let j = 0; j < rHat; j++) mPrime += params.decay ** j;
|
||||
const mPrime = usePotential(rHat, params.decay);
|
||||
const value = params.u * mPrime * voi;
|
||||
const cost = 2 * Math.max(0, g0) * ((K + size) / (K + size + 1) - K / (K + 1));
|
||||
const cost = runCost(g0, K, size);
|
||||
combos.push({ factors: S.map((c) => model.factors[c]!), voi, rHat, mPrime, value, cost, net: value - cost });
|
||||
}
|
||||
}
|
||||
|
||||
@@ -23,6 +23,15 @@ export function programProblems(inc: Incoming, protocol: ProtocolBody | null): s
|
||||
...factorProblems(inc.content, inc.checklist),
|
||||
...increasing("authorEstimate", inc.content.authorEstimate),
|
||||
];
|
||||
case "diagnosisProposal":
|
||||
return diagnosisProposalProblems(inc.content, inc.checklist);
|
||||
case "diagnosisPrediction":
|
||||
return diagnosisPredictionProblems(inc.content, inc.checklist);
|
||||
case "diagnosisImplementation":
|
||||
case "diagnosisRepair":
|
||||
return codeProblems(inc.known.code);
|
||||
case "diagnosisVerification":
|
||||
return covers("experiments 的实验", inc.content.experiments.map((x) => x.experiment), inc.checklist.experiments);
|
||||
case "firstPrediction":
|
||||
case "secondPrediction":
|
||||
return increasing("range", inc.content.range);
|
||||
@@ -63,6 +72,45 @@ function claimProblems(c: Submission<"claim">, k: ChecklistOf<"claim">): string[
|
||||
];
|
||||
}
|
||||
|
||||
/**
|
||||
* 诊断解释与诊断实验(方法.md 4.3 诊 1、第五部分"指标"):解释 1 至 M 条、实验 1 至 A 个(下限由字段定义管);
|
||||
* 诊断实验登记的指标只能选"缺了只记缺失";同一个实验里指标短名不重名、不占用已有编号
|
||||
* (不同实验的同名短名冻结时各带自己实验的前缀,不会撞)。
|
||||
*/
|
||||
function diagnosisProposalProblems(c: Submission<"diagnosisProposal">, k: ChecklistOf<"diagnosisProposal">): string[] {
|
||||
return [
|
||||
...(c.explanations.length > k.maxExplanations ? [`交了 ${c.explanations.length} 条诊断解释,最多 ${k.maxExplanations} 条`] : []),
|
||||
...(c.experiments.length > k.maxExperiments ? [`交了 ${c.experiments.length} 个诊断实验,最多 ${k.maxExperiments} 个`] : []),
|
||||
...c.experiments.flatMap((e, i) => {
|
||||
const ids = e.metrics.map((m) => m.id);
|
||||
return [
|
||||
...e.metrics
|
||||
.filter((m) => m.missing !== "recordMissing")
|
||||
.map((m) => `第 ${i + 1} 个实验的指标 ${m.id} 选了 ${m.missing}:诊断实验里的指标只能选 recordMissing(缺了只记缺失)`),
|
||||
...repeated(ids).map((x) => `第 ${i + 1} 个实验的 metrics 里编号 ${x} 写了不止一次`),
|
||||
...unique(ids)
|
||||
.filter((x) => k.takenMetricIds.includes(x))
|
||||
.map((x) => `第 ${i + 1} 个实验的指标编号 ${x} 已被占用(已占用的是 ${k.takenMetricIds.join("、")}):换一个编号`),
|
||||
];
|
||||
}),
|
||||
];
|
||||
}
|
||||
|
||||
/**
|
||||
* 诊断预测(方法.md 4.3 诊 2):覆盖自己每条解释 × 每个合并后的实验,不多不少、每一对一次;每个范围三个数严格递增。
|
||||
* 一对写成"解释编号:实验编号"(编号里没有冒号)。
|
||||
*/
|
||||
function diagnosisPredictionProblems(c: Submission<"diagnosisPrediction">, k: ChecklistOf<"diagnosisPrediction">): string[] {
|
||||
return [
|
||||
...covers(
|
||||
"predictions 的(解释:实验)",
|
||||
c.predictions.map((p) => `${p.explanation}:${p.experiment}`),
|
||||
k.explanations.flatMap((e) => k.experiments.map((d) => `${e}:${d}`)),
|
||||
),
|
||||
...c.predictions.flatMap((p) => increasing(`${p.explanation}:${p.experiment} 的 range`, p.range)),
|
||||
];
|
||||
}
|
||||
|
||||
/**
|
||||
* 提案、回应、统一共用(方法.md 第五部分"延续""指标"、4.3 ②⑥):
|
||||
* 规则 2 因子数上限;3 两个因子不延续同一个、延续的编号看得到;4 保障短名;10 指标的引用与新登记。
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
// submissions:一份提交收下还是退回,先程序检查、再问 Jev(docs/结构.md 第二部分);
|
||||
// 另有冻结协议的拼法,和核验、消融核验收下之后通过没有。不读也不写账本。
|
||||
export { ablationOutcome, verificationOutcome } from "./outcomes.ts";
|
||||
export { ablationOutcome, diagnosisOutcome, verificationOutcome } from "./outcomes.ts";
|
||||
export { protocolBody } from "./protocol.ts";
|
||||
export { receive } from "./receive.ts";
|
||||
export {
|
||||
|
||||
@@ -34,3 +34,20 @@ export function ablationOutcome(
|
||||
const wronglyOff = v.factors.filter((f) => !shouldBeOff.has(f.factor) && f.off).map((f) => f.factor);
|
||||
return { passed: leftOn.length + wronglyOff.length === 0, leftOn, wronglyOff };
|
||||
}
|
||||
|
||||
/**
|
||||
* 诊断核验通过没有(方法.md 4.3 诊 4):逐个实验判,任何一项写了 false 的实验不通过。
|
||||
* notFollowed 是没照写法做的实验,touchedElse 是动了写法以外的地方的实验,failing 是两者的并集,按核验里的次序。
|
||||
* 一个实验不通过不影响别的实验:通过的实验代码不再改动。
|
||||
*/
|
||||
export function diagnosisOutcome(v: Submission<"diagnosisVerification">): {
|
||||
passed: boolean;
|
||||
failing: string[];
|
||||
notFollowed: string[];
|
||||
touchedElse: string[];
|
||||
} {
|
||||
const notFollowed = v.experiments.filter((x) => !x.followed).map((x) => x.experiment);
|
||||
const touchedElse = v.experiments.filter((x) => !x.nothingElse).map((x) => x.experiment);
|
||||
const failing = v.experiments.filter((x) => !x.followed || !x.nothingElse).map((x) => x.experiment);
|
||||
return { passed: failing.length === 0, failing, notFollowed, touchedElse };
|
||||
}
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
import { checkHunk, checkProtocol, type CodeHunk } from "../jev/index.ts";
|
||||
import { checkExperiment, checkHunk, checkProtocol, type CodeHunk } from "../jev/index.ts";
|
||||
import type { JevJudgment } from "../vocab/index.ts";
|
||||
import { programProblems } from "./checks.ts";
|
||||
import { protocolBody } from "./protocol.ts";
|
||||
@@ -27,7 +27,8 @@ export async function receive(inc: Incoming): Promise<Verdict> {
|
||||
}
|
||||
|
||||
/**
|
||||
* 问 Jev(方法.md 4.3 ⑥⑨):回应、统一对拼好的协议问冻结条款三问;实现、修复每个差异块问代码改动五问,顺序发。
|
||||
* 问 Jev(方法.md 4.3 ⑥⑨、诊 4):回应、统一对拼好的协议问冻结条款三问;实现、修复(含诊断程序)每个差异块问代码改动五问,顺序发。
|
||||
* 诊断解释与实验(诊 1)对每个实验问冻结条款三问(改动了评测、自相矛盾、太含糊,用诊断的变体问法),顺序发。
|
||||
* 没有差异块的改动(二进制、只改可执行位、空文件增删)Jev 看不到内容,不问;一块都没有就等于没问,交回 null。
|
||||
* 消融实现不问:冻结协议里没有登记"关掉以后"的随机性和停止条件,五问没有对照(照参考项目)。
|
||||
* flagged 是退回原因里每一处的文字:只列判"是"的几问,问题名用 jev 的英文名(用户 2026-10-03 定);判"是"只看 jev 交回的 yes。
|
||||
@@ -37,7 +38,19 @@ async function askJev(inc: Incoming, protocol: ProtocolBody | null): Promise<{ j
|
||||
const c = await checkProtocol(inc.jevModel, protocol!);
|
||||
return { judgments: c.judgments, flagged: c.flagged ? [`- 冻结协议:${flaggedText(c.judgments)}`] : [] };
|
||||
}
|
||||
if (inc.kind !== "implementation" && inc.kind !== "repair") return null;
|
||||
if (inc.kind === "diagnosisProposal") {
|
||||
const judgments: JevJudgment[] = [];
|
||||
const flagged: string[] = [];
|
||||
for (const [i, experiment] of inc.content.experiments.entries()) {
|
||||
const c = await checkExperiment(inc.jevModel, experiment);
|
||||
judgments.push(...c.judgments);
|
||||
if (c.flagged) flagged.push(`- 第 ${i + 1} 个诊断实验(${experiment.name}):${flaggedText(c.judgments)}`);
|
||||
}
|
||||
return { judgments, flagged };
|
||||
}
|
||||
if (inc.kind !== "implementation" && inc.kind !== "repair" && inc.kind !== "diagnosisImplementation" && inc.kind !== "diagnosisRepair") {
|
||||
return null;
|
||||
}
|
||||
// 走到这里程序检查已过,代码一定存成了版本。
|
||||
const code = inc.known.code as Extract<typeof inc.known.code, { saved: true }>;
|
||||
if (code.hunks.length === 0) return null;
|
||||
|
||||
@@ -17,7 +17,7 @@ export type OriginalProposal = Pick<Submission<"proposal">, "factors" | "safegua
|
||||
type NotSaved = { saved: false; path: string; reason: string };
|
||||
|
||||
/**
|
||||
* 实现、修复的代码:存成了版本就带改了的不许改的文件(tasks 的 protectedChanges)、
|
||||
* 实现、修复(候选和诊断程序同样)的代码:存成了版本就带改了的不许改的文件(tasks 的 protectedChanges)、
|
||||
* 和本批基线比出的差异块(jev 的 codeHunks 切好的),以及给 Jev 对照的协议正文和要采集的指标。
|
||||
*/
|
||||
export type ForwardCode = NotSaved | { saved: true; protectedFiles: string[]; hunks: CodeHunk[]; protocol: ProtocolRules };
|
||||
@@ -33,6 +33,9 @@ type KnownByKind = {
|
||||
response: { original: OriginalProposal };
|
||||
implementation: { code: ForwardCode };
|
||||
repair: { code: ForwardCode };
|
||||
/** 诊断程序和候选代码同形:和当前基线比出的差异块,给 Jev 对照的是选中实验的原文和要采集的指标。 */
|
||||
diagnosisImplementation: { code: ForwardCode };
|
||||
diagnosisRepair: { code: ForwardCode };
|
||||
/** 交了"跑不起来"(cannotRun 不是 null)时 flow 不存代码,code 为 null。 */
|
||||
ablationImplementation: { code: AblationCode | null };
|
||||
};
|
||||
|
||||
+227
-13
@@ -1,4 +1,4 @@
|
||||
import { copyFileSync, cpSync, existsSync, readFileSync, writeFileSync } from "node:fs";
|
||||
import { chmodSync, copyFileSync, cpSync, existsSync, mkdirSync, readFileSync, writeFileSync } from "node:fs";
|
||||
import { join } from "node:path";
|
||||
import { Type, type Static } from "typebox";
|
||||
import { Value } from "typebox/value";
|
||||
@@ -13,7 +13,16 @@ import type {
|
||||
} from "../vocab/index.ts";
|
||||
import type { TaskFacts } from "./facts.ts";
|
||||
import { measured, readMetricsFile, unmeasured } from "./metrics-file.ts";
|
||||
import type { LightCheckOutcome, Task, TaskImplementation } from "./types.ts";
|
||||
import type {
|
||||
DiagnosisEnding,
|
||||
DiagnosisExperiment,
|
||||
DiagnosisExperimentJudged,
|
||||
DiagnosisJudged,
|
||||
DiagnosisJudgingStep,
|
||||
LightCheckOutcome,
|
||||
Task,
|
||||
TaskImplementation,
|
||||
} from "./types.ts";
|
||||
|
||||
// 填圆:在单位正方形里放 26 个圆,最大化半径之和。任务说明见 tasks/circle_packing/TASK.md。
|
||||
// 凡是要执行任务代码的事(正式运行、校验器、轻量检查)都只交出运行定义,由调用方交给 runner 在任务镜像里跑;
|
||||
@@ -22,6 +31,7 @@ import type { LightCheckOutcome, Task, TaskImplementation } from "./types.ts";
|
||||
const CIRCLES = 26;
|
||||
const ENTRY = "initial.py";
|
||||
const HARNESS = "harness.py";
|
||||
const DIAGNOSIS_HARNESS = "harness-diagnosis.py";
|
||||
const VALIDATOR = "validate.py";
|
||||
const CHECK_SCRIPT = "check-candidate.py";
|
||||
const METRICS_SPEC = "metrics-spec.json";
|
||||
@@ -29,6 +39,12 @@ const METRICS = "metrics.json";
|
||||
const SOLUTION = "solution.json";
|
||||
const VERDICT = "verdict.json";
|
||||
const CHECK_RESULT = "check.json";
|
||||
/** 诊断运行:原始产物目录、每个实验的答案目录、harness 写的每实验用时报告(路径都相对运行目录)。 */
|
||||
const BASELINE_ARTIFACT = "baseline-artifact";
|
||||
const ANSWERS = "answers";
|
||||
const DIAGNOSIS_REPORT = "diagnosis.json";
|
||||
/** 判分目录里每个实验写的校验器退出码文件(同一个目录下还有这个实验的 solution.json、verdict.json)。 */
|
||||
const EXIT_FILE = "exit";
|
||||
/** 轻量检查目录里放候选代码的子目录;检查脚本放在它的上一层,免得扫到自己。 */
|
||||
const CHECK_CANDIDATE = "candidate";
|
||||
/** 运行目录挂进容器的位置。 */
|
||||
@@ -277,16 +293,16 @@ function readVerdict(judgeDir: string, exitCode: number): ValidatorVerdict {
|
||||
return verdict;
|
||||
}
|
||||
|
||||
function judge(
|
||||
_task: Task,
|
||||
runDir: string,
|
||||
judgeDir: string,
|
||||
formal: ProcessResult,
|
||||
validator: ProcessResult,
|
||||
hypothesisMetrics: HypothesisMetric[],
|
||||
): JudgedResult {
|
||||
const run = started(formal);
|
||||
const verdict = readVerdict(judgeDir, requireExit("校验器", validator, [0, 1]));
|
||||
/** 校验器判定读成的结论:算不算数、理由、最终分数、校验器给的辅助指标、合法化明细。 */
|
||||
interface Scored {
|
||||
valid: boolean;
|
||||
reason: string;
|
||||
score: Measured;
|
||||
fromValidator: Record<string, Measured>;
|
||||
extra: Json;
|
||||
}
|
||||
|
||||
function scoreVerdict(verdict: ValidatorVerdict): Scored {
|
||||
const legal = verdict.legalization;
|
||||
let deduction: Measured;
|
||||
let score: Measured;
|
||||
@@ -328,7 +344,199 @@ function judge(
|
||||
deduction,
|
||||
},
|
||||
};
|
||||
return assemble(runDir, run, hypothesisMetrics, { valid: verdict.valid, reason }, score, fromValidator, extra);
|
||||
return { valid: verdict.valid, reason, score, fromValidator, extra };
|
||||
}
|
||||
|
||||
function judge(
|
||||
_task: Task,
|
||||
runDir: string,
|
||||
judgeDir: string,
|
||||
formal: ProcessResult,
|
||||
validator: ProcessResult,
|
||||
hypothesisMetrics: HypothesisMetric[],
|
||||
): JudgedResult {
|
||||
const run = started(formal);
|
||||
const scored = scoreVerdict(readVerdict(judgeDir, requireExit("校验器", validator, [0, 1])));
|
||||
return assemble(runDir, run, hypothesisMetrics, { valid: scored.valid, reason: scored.reason }, scored.score, scored.fromValidator, scored.extra);
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// 诊断运行(方法.md 4.3"诊断"一段):一个容器、一次登记、一次扣次,里面按顺序跑几个实验。
|
||||
// 全部是新增的一套函数,用自己的 harness(harness-diagnosis.py),不碰上面正式运行的任何一步。
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
/** 诊断程序存版本时跳过的名字:正式运行那份加上原始产物目录(它是宿主放进工作目录的,不是实验员的代码)。 */
|
||||
function diagnosisUnversionedNames(facts: TaskFacts): string[] {
|
||||
return [...facts.unversionedNames, BASELINE_ARTIFACT];
|
||||
}
|
||||
|
||||
/**
|
||||
* 把产出当前基线的那次正式运行的原始产物(solution.json)放进 dir 下的 baseline-artifact/,设成只读。
|
||||
* 实验员的工作目录和诊断运行的运行目录都用它:实验员能看到实验要接着算的那份答案,运行时实验读的是同一份。
|
||||
* 只读靠文件权限,防的是实验顺手改了它,不是安全边界。
|
||||
*/
|
||||
function placeBaselineArtifact(_task: Task, baselineRunDir: string, dir: string): void {
|
||||
const target = join(dir, BASELINE_ARTIFACT);
|
||||
mkdirSync(target);
|
||||
const file = join(target, SOLUTION);
|
||||
copyFileSync(join(baselineRunDir, SOLUTION), file);
|
||||
chmodSync(file, 0o444);
|
||||
}
|
||||
|
||||
function prepareDiagnosisRun(task: Task, programDir: string, baselineRunDir: string, runDir: string, experiments: DiagnosisExperiment[]): void {
|
||||
cpSync(programDir, runDir, { recursive: true });
|
||||
// 两份 harness 以宿主的为准;指标说明是全部实验登记的指标合在一起,每个实验只写自己的那几条。
|
||||
copyFileSync(join(task.dir, HARNESS), join(runDir, HARNESS));
|
||||
copyFileSync(join(task.dir, DIAGNOSIS_HARNESS), join(runDir, DIAGNOSIS_HARNESS));
|
||||
writeFileSync(join(runDir, METRICS_SPEC), `${JSON.stringify(experiments.flatMap((e) => e.metrics), null, 2)}\n`);
|
||||
placeBaselineArtifact(task, baselineRunDir, runDir);
|
||||
mkdirSync(join(runDir, ANSWERS));
|
||||
}
|
||||
|
||||
/** 总时限、核数照任务规定,实验按顺序跑,每个实验分到总时限 ÷ 实验数(由 harness 执行)。 */
|
||||
function diagnosisRun(task: Task, imageDigest: string, seed: string, experimentIds: string[]): RunDefinition {
|
||||
const eachSeconds = task.facts.run.wallSeconds / experimentIds.length;
|
||||
const command = [
|
||||
"python",
|
||||
"-B",
|
||||
`${WORKDIR}/${DIAGNOSIS_HARNESS}`,
|
||||
"--program",
|
||||
ENTRY,
|
||||
"--baseline",
|
||||
BASELINE_ARTIFACT,
|
||||
"--answers",
|
||||
ANSWERS,
|
||||
"--seconds-each",
|
||||
String(eachSeconds),
|
||||
"--report",
|
||||
DIAGNOSIS_REPORT,
|
||||
"--experiments",
|
||||
...experimentIds,
|
||||
];
|
||||
return definition(task, imageDigest, command, task.facts.run.wallSeconds, { INFORSI_SEED: seed });
|
||||
}
|
||||
|
||||
/** 一个实验在运行目录里的结局:harness 报告里的一项,或者整次运行先结束、没轮到它。 */
|
||||
type ExperimentOutcome = { ending: "finished" | "crashed" | "timeout"; wallSeconds: number } | { ending: "notRun"; reason: string };
|
||||
|
||||
/** harness 写的报告(我们自己的程序,按约定的格式直接读)。 */
|
||||
interface ReportEntry {
|
||||
id: string;
|
||||
status: "finished" | "crashed" | "timeout";
|
||||
wall_seconds: number;
|
||||
}
|
||||
|
||||
function experimentOutcomes(
|
||||
task: Task,
|
||||
runDir: string,
|
||||
process: Exclude<ProcessResult, { ending: "notStarted" }>,
|
||||
experiments: DiagnosisExperiment[],
|
||||
): ExperimentOutcome[] {
|
||||
const path = join(runDir, DIAGNOSIS_REPORT);
|
||||
const report = existsSync(path) ? (readJson(path) as ReportEntry[]) : [];
|
||||
return experiments.map((e) => {
|
||||
const entry = report.find((r) => r.id === e.id);
|
||||
if (entry !== undefined) return { ending: entry.status, wallSeconds: entry.wall_seconds };
|
||||
const failure = processFailure(task, process);
|
||||
if (failure === null) throw new Error(`诊断运行正常退出,但 ${DIAGNOSIS_REPORT} 里没有实验 ${e.id}`);
|
||||
return { ending: "notRun", reason: `没轮到:整次诊断运行${failure}` };
|
||||
});
|
||||
}
|
||||
|
||||
/** 没有答案可判的实验:不算数,辅助指标和假设专用指标都记测不到并附原因。 */
|
||||
function unjudgedExperiment(
|
||||
id: string,
|
||||
ending: Exclude<DiagnosisEnding, "finished">,
|
||||
wall: Measured,
|
||||
reason: string,
|
||||
hypothesisMetrics: HypothesisMetric[],
|
||||
): DiagnosisExperimentJudged {
|
||||
const metrics = {
|
||||
...Object.fromEntries(FROM_VALIDATOR.map((m) => [m, unmeasured(`没有跑校验器:${reason}`)])),
|
||||
...Object.fromEntries(hypothesisMetrics.map((m) => [m.id, unmeasured(`没有结果:${reason}`)])),
|
||||
};
|
||||
return { id, ending, wallSeconds: wall, verdict: "invalid", verdictReason: reason, score: unmeasured(reason), metrics, extra: { legalization: null } };
|
||||
}
|
||||
|
||||
function judgedExperiments(
|
||||
task: Task,
|
||||
runDir: string,
|
||||
judgeDir: string,
|
||||
experiments: DiagnosisExperiment[],
|
||||
outcomes: ExperimentOutcome[],
|
||||
): DiagnosisJudged {
|
||||
const eachSeconds = task.facts.run.wallSeconds / experiments.length;
|
||||
return {
|
||||
experiments: experiments.map((e, i): DiagnosisExperimentJudged => {
|
||||
const outcome = outcomes[i]!;
|
||||
if (outcome.ending === "notRun") return unjudgedExperiment(e.id, "notRun", unmeasured(outcome.reason), outcome.reason, e.metrics);
|
||||
const wall = measured(outcome.wallSeconds);
|
||||
if (outcome.ending === "timeout") {
|
||||
return unjudgedExperiment(e.id, "timeout", wall, `超过分到的时间 ${eachSeconds} 秒,被结束`, e.metrics);
|
||||
}
|
||||
if (outcome.ending === "crashed") {
|
||||
return unjudgedExperiment(e.id, "crashed", wall, `实验的进程以非零退出码结束(报错见这次运行的 stderr.log)`, e.metrics);
|
||||
}
|
||||
const dir = join(judgeDir, e.id);
|
||||
const exitCode = Number(readFileSync(join(dir, EXIT_FILE), "utf8").trim());
|
||||
const scored = scoreVerdict(readVerdict(dir, exitCode));
|
||||
const hypothesis = readMetricsFile(join(runDir, ANSWERS, e.id, METRICS), `${ANSWERS}/${e.id}/${METRICS}`, e.metrics).hypothesis;
|
||||
return {
|
||||
id: e.id,
|
||||
ending: "finished",
|
||||
wallSeconds: wall,
|
||||
verdict: scored.valid ? "valid" : "invalid",
|
||||
verdictReason: scored.reason,
|
||||
score: scored.score,
|
||||
metrics: { ...scored.fromValidator, ...hypothesis },
|
||||
extra: scored.extra,
|
||||
};
|
||||
}),
|
||||
};
|
||||
}
|
||||
|
||||
/** 逐个实验判分。校验器一个容器、按顺序对每个有答案的实验各跑一次,退出码各存一个文件。 */
|
||||
const VALIDATE_EACH = `for id in "$@"; do
|
||||
python -B ${VALIDATOR} --circles ${CIRCLES} --solution "$id/${SOLUTION}" --atol ${TOLERANCE} --legalize --verdict "$id/${VERDICT}"
|
||||
echo $? > "$id/${EXIT_FILE}"
|
||||
done`;
|
||||
|
||||
function prepareDiagnosisJudging(
|
||||
task: Task,
|
||||
runDir: string,
|
||||
judgeDir: string,
|
||||
imageDigest: string,
|
||||
process: ProcessResult,
|
||||
experiments: DiagnosisExperiment[],
|
||||
): DiagnosisJudgingStep {
|
||||
const run = started(process);
|
||||
const outcomes = experimentOutcomes(task, runDir, run, experiments);
|
||||
const finished = experiments.filter((_, i) => outcomes[i]!.ending === "finished");
|
||||
if (finished.length === 0) return { status: "judged", result: judgedExperiments(task, runDir, judgeDir, experiments, outcomes) };
|
||||
for (const e of finished) {
|
||||
mkdirSync(join(judgeDir, e.id));
|
||||
copyFileSync(join(runDir, ANSWERS, e.id, SOLUTION), join(judgeDir, e.id, SOLUTION));
|
||||
}
|
||||
copyFileSync(join(task.dir, VALIDATOR), join(judgeDir, VALIDATOR));
|
||||
const command = ["sh", "-c", VALIDATE_EACH, "inforsi-validate-each", ...finished.map((e) => e.id)];
|
||||
return { status: "validate", definition: definition(task, imageDigest, command, VALIDATOR_SECONDS * finished.length, {}) };
|
||||
}
|
||||
|
||||
function judgeDiagnosis(
|
||||
task: Task,
|
||||
runDir: string,
|
||||
judgeDir: string,
|
||||
process: ProcessResult,
|
||||
validator: ProcessResult | null,
|
||||
experiments: DiagnosisExperiment[],
|
||||
): DiagnosisJudged {
|
||||
const run = started(process);
|
||||
const outcomes = experimentOutcomes(task, runDir, run, experiments);
|
||||
if (outcomes.some((o) => o.ending === "finished")) {
|
||||
if (validator === null) throw new Error("有实验交出了答案,却没有校验器的运行结果");
|
||||
requireExit("校验器", validator, [0]);
|
||||
}
|
||||
return judgedExperiments(task, runDir, judgeDir, experiments, outcomes);
|
||||
}
|
||||
|
||||
export const circlePacking: TaskImplementation = {
|
||||
@@ -339,4 +547,10 @@ export const circlePacking: TaskImplementation = {
|
||||
readLightCheck,
|
||||
prepareJudging,
|
||||
judge,
|
||||
diagnosisUnversionedNames,
|
||||
placeBaselineArtifact,
|
||||
prepareDiagnosisRun,
|
||||
diagnosisRun,
|
||||
prepareDiagnosisJudging,
|
||||
judgeDiagnosis,
|
||||
};
|
||||
|
||||
+16
-1
@@ -2,14 +2,29 @@
|
||||
// 不起子进程:要执行任务代码的事都只交出运行定义,由调用方交给 runner 在任务镜像里跑。
|
||||
export { checkTaskFacts, TASK_KINDS, TaskFacts, type TaskKind } from "./facts.ts";
|
||||
export {
|
||||
diagnosisRun,
|
||||
diagnosisUnversionedNames,
|
||||
formalRun,
|
||||
judge,
|
||||
judgeDiagnosis,
|
||||
lightCheck,
|
||||
loadTask,
|
||||
metricDefinitions,
|
||||
placeBaselineArtifact,
|
||||
prepareDiagnosisJudging,
|
||||
prepareDiagnosisRun,
|
||||
prepareJudging,
|
||||
prepareRun,
|
||||
protectedChanges,
|
||||
readLightCheck,
|
||||
} from "./task.ts";
|
||||
export type { JudgingStep, LightCheckOutcome, Task } from "./types.ts";
|
||||
export type {
|
||||
DiagnosisEnding,
|
||||
DiagnosisExperiment,
|
||||
DiagnosisExperimentJudged,
|
||||
DiagnosisJudged,
|
||||
DiagnosisJudgingStep,
|
||||
JudgingStep,
|
||||
LightCheckOutcome,
|
||||
Task,
|
||||
} from "./types.ts";
|
||||
|
||||
+80
-1
@@ -3,7 +3,15 @@ import { join, resolve } from "node:path";
|
||||
import type { HypothesisMetric, JudgedResult, MetricDefinition, ProcessResult, RunDefinition } from "../vocab/index.ts";
|
||||
import { circlePacking } from "./circle-packing.ts";
|
||||
import { checkTaskFacts, type TaskKind } from "./facts.ts";
|
||||
import type { JudgingStep, LightCheckOutcome, Task, TaskImplementation } from "./types.ts";
|
||||
import type {
|
||||
DiagnosisExperiment,
|
||||
DiagnosisJudged,
|
||||
DiagnosisJudgingStep,
|
||||
JudgingStep,
|
||||
LightCheckOutcome,
|
||||
Task,
|
||||
TaskImplementation,
|
||||
} from "./types.ts";
|
||||
|
||||
// tasks 对外的函数。任务目录的约定:task.json(任务事实)、TASK.md(给实验员的说明)、
|
||||
// Dockerfile 与依赖清单(建镜像)、baseline/(基线代码),以及各任务自己的 Python 工具。
|
||||
@@ -105,3 +113,74 @@ export function judge(
|
||||
export function protectedChanges(task: Task, changedFiles: string[]): string[] {
|
||||
return changedFiles.filter((file) => task.facts.protectedFiles.includes(file));
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// 诊断运行(方法.md 4.3"诊断"一段):一个容器、一次登记、一次扣次,里面按顺序跑几个实验。
|
||||
// 是和上面正式运行并列的另一套函数,上面的函数不认识它。各目录同样由调用方建好、传空目录进来。
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
/** 存诊断程序版本时跳过的名字:task.json 的清单,加上宿主放进工作目录的原始产物目录。 */
|
||||
export function diagnosisUnversionedNames(task: Task): string[] {
|
||||
return implementation(task).diagnosisUnversionedNames(task.facts);
|
||||
}
|
||||
|
||||
/**
|
||||
* 把产出当前基线的那次正式运行的原始产物,只读地放进 dir(实验员工作目录,调用方已用 checkoutVersion 取出基线代码)。
|
||||
* baselineRunDir 是那次运行的运行目录。放在哪、叫什么,由任务定,写在 TASK.md 里给实验员看。
|
||||
*/
|
||||
export function placeBaselineArtifact(task: Task, baselineRunDir: string, dir: string): void {
|
||||
implementation(task).placeBaselineArtifact(task, baselineRunDir, dir);
|
||||
}
|
||||
|
||||
/**
|
||||
* 准备一次诊断运行的运行目录:诊断程序、两份 harness、全部实验登记的指标说明(metrics-spec.json)、
|
||||
* 只读的原始产物。runDir 由调用方建好,应为空目录;实验的先后顺序就是 experiments 的顺序。
|
||||
*/
|
||||
export function prepareDiagnosisRun(
|
||||
task: Task,
|
||||
programDir: string,
|
||||
baselineRunDir: string,
|
||||
runDir: string,
|
||||
experiments: DiagnosisExperiment[],
|
||||
): void {
|
||||
implementation(task).prepareDiagnosisRun(task, programDir, baselineRunDir, runDir, experiments);
|
||||
}
|
||||
|
||||
/**
|
||||
* 诊断运行的运行定义:总时限、核数照任务规定,harness 按顺序跑 experimentIds 里的实验,
|
||||
* 每个实验分到总时限 ÷ 实验数。种子同正式运行(同一份研究共用一个)。
|
||||
*/
|
||||
export function diagnosisRun(task: Task, imageDigest: string, seed: string, experimentIds: string[]): RunDefinition {
|
||||
return implementation(task).diagnosisRun(task, imageDigest, seed, experimentIds);
|
||||
}
|
||||
|
||||
/**
|
||||
* 诊断判分第一步:每个交出答案的实验的答案复制进 judgeDir 下以实验编号命名的子目录,
|
||||
* 交回校验器的运行定义(一个容器逐个实验判分);没有一个实验交出答案时直接交回判分结果。
|
||||
*/
|
||||
export function prepareDiagnosisJudging(
|
||||
task: Task,
|
||||
runDir: string,
|
||||
judgeDir: string,
|
||||
imageDigest: string,
|
||||
process: ProcessResult,
|
||||
experiments: DiagnosisExperiment[],
|
||||
): DiagnosisJudgingStep {
|
||||
return implementation(task).prepareDiagnosisJudging(task, runDir, judgeDir, imageDigest, process, experiments);
|
||||
}
|
||||
|
||||
/**
|
||||
* 诊断判分第二步:按实验分开交回分数、合法化、登记的指标、用时、结局。validator 是第一步交回过运行定义时跑出的结果,
|
||||
* 第一步直接判完时传 null。资源用量记在整次运行的 ProcessResult 里,不在这里。
|
||||
* 校验器自己没正常跑完、判定文件认不出、修后分数高于原分数时报错停下,同正式运行的 judge。
|
||||
*/
|
||||
export function judgeDiagnosis(
|
||||
task: Task,
|
||||
runDir: string,
|
||||
judgeDir: string,
|
||||
process: ProcessResult,
|
||||
validator: ProcessResult | null,
|
||||
experiments: DiagnosisExperiment[],
|
||||
): DiagnosisJudged {
|
||||
return implementation(task).judgeDiagnosis(task, runDir, judgeDir, process, validator, experiments);
|
||||
}
|
||||
|
||||
+60
-1
@@ -1,4 +1,4 @@
|
||||
import type { HypothesisMetric, JudgedResult, MetricDefinition, ProcessResult, RunDefinition } from "../vocab/index.ts";
|
||||
import type { HypothesisMetric, JudgedResult, Json, Measured, MetricDefinition, ProcessResult, RunDefinition, Verdict } from "../vocab/index.ts";
|
||||
import type { TaskFacts } from "./facts.ts";
|
||||
|
||||
/** 读进来的一个任务。 */
|
||||
@@ -22,6 +22,44 @@ export type LightCheckOutcome = { passed: true } | { passed: false; reasons: str
|
||||
*/
|
||||
export type JudgingStep = { status: "judged"; result: JudgedResult } | { status: "validate"; definition: RunDefinition };
|
||||
|
||||
/** 一个要跑的诊断实验:编号,以及它自己登记的假设专用指标(冻结后的全名;诊断实验只能登记"缺了只记缺失"的指标)。 */
|
||||
export interface DiagnosisExperiment {
|
||||
id: string;
|
||||
metrics: HypothesisMetric[];
|
||||
}
|
||||
|
||||
/**
|
||||
* 一个诊断实验怎么结束:finished 跑完交出了答案;crashed 实验自己崩了;timeout 超过分到的时间被杀;
|
||||
* notRun 整次运行先被总时限或从外面结束,没轮到它。
|
||||
*/
|
||||
export type DiagnosisEnding = "finished" | "crashed" | "timeout" | "notRun";
|
||||
|
||||
/** 一个诊断实验单独判分的结果。资源用量(CPU、峰值内存)记在整次运行的结果里,这里只有它自己的用时。 */
|
||||
export interface DiagnosisExperimentJudged {
|
||||
id: string;
|
||||
ending: DiagnosisEnding;
|
||||
/** 这个实验实际用掉的墙钟秒数;notRun 时测不到。 */
|
||||
wallSeconds: Measured;
|
||||
verdict: Verdict;
|
||||
verdictReason: string;
|
||||
/** 合法化之后的最终指标分数;不算数时测不到并附原因。 */
|
||||
score: Measured;
|
||||
/** 校验器给的辅助指标,加上这个实验登记的假设专用指标,键是指标编号。 */
|
||||
metrics: Record<string, Measured>;
|
||||
/** 同正式运行的 extra:填圆是合法化的明细。 */
|
||||
extra: Json;
|
||||
}
|
||||
|
||||
/** 一次诊断运行的判分结果:每个实验一项,顺序同运行定义里的实验顺序。 */
|
||||
export interface DiagnosisJudged {
|
||||
experiments: DiagnosisExperimentJudged[];
|
||||
}
|
||||
|
||||
/** 诊断判分第一步:不用跑校验器就已判定(没有一个实验交出答案),或交回校验器的运行定义。 */
|
||||
export type DiagnosisJudgingStep =
|
||||
| { status: "judged"; result: DiagnosisJudged }
|
||||
| { status: "validate"; definition: RunDefinition };
|
||||
|
||||
/** 一种任务的实现。对外函数按 task.facts.kind 从对照表里取出它来调。参数的含义见 task.ts 里同名的对外函数。 */
|
||||
export interface TaskImplementation {
|
||||
/** 最终指标(编号和方向取自 task.json)与常规辅助指标的定义。 */
|
||||
@@ -46,4 +84,25 @@ export interface TaskImplementation {
|
||||
validator: ProcessResult,
|
||||
hypothesisMetrics: HypothesisMetric[],
|
||||
): JudgedResult;
|
||||
/** 诊断运行(方法.md 4.3"诊断"一段):一次运行跑几个实验。上面七个函数的行为与它无关。 */
|
||||
diagnosisUnversionedNames(facts: TaskFacts): string[];
|
||||
placeBaselineArtifact(task: Task, baselineRunDir: string, dir: string): void;
|
||||
prepareDiagnosisRun(task: Task, programDir: string, baselineRunDir: string, runDir: string, experiments: DiagnosisExperiment[]): void;
|
||||
diagnosisRun(task: Task, imageDigest: string, seed: string, experimentIds: string[]): RunDefinition;
|
||||
prepareDiagnosisJudging(
|
||||
task: Task,
|
||||
runDir: string,
|
||||
judgeDir: string,
|
||||
imageDigest: string,
|
||||
process: ProcessResult,
|
||||
experiments: DiagnosisExperiment[],
|
||||
): DiagnosisJudgingStep;
|
||||
judgeDiagnosis(
|
||||
task: Task,
|
||||
runDir: string,
|
||||
judgeDir: string,
|
||||
process: ProcessResult,
|
||||
validator: ProcessResult | null,
|
||||
experiments: DiagnosisExperiment[],
|
||||
): DiagnosisJudged;
|
||||
}
|
||||
|
||||
@@ -22,6 +22,9 @@ type FactorRules = {
|
||||
/** 实现、修复、核验共用:冻结协议的实现条款(c1…)、因子(p1…)、保障短名。 */
|
||||
type FrozenItems = { candidate: string; clauses: string[]; factors: string[]; safeguards: string[] };
|
||||
|
||||
/** 诊断实现、修复、核验共用:本次诊断选中的实验编号(d1…)。 */
|
||||
type DiagnosisItems = { experiments: string[] };
|
||||
|
||||
/** 入口个数的范围(方法.md 4.2:2–3 个)。 */
|
||||
type EntryCount = { min: 2; max: 3 };
|
||||
|
||||
@@ -30,6 +33,16 @@ export type Checklist =
|
||||
| { kind: "discussion"; entries: EntryCount }
|
||||
/** minChoices = min(N, 入口总数)(方法.md 4.2 认领)。 */
|
||||
| { kind: "claim"; entries: string[]; minChoices: number }
|
||||
/**
|
||||
* 诊断解释最多 M 条、诊断实验最多 A 个(方法.md 4.3 诊 1);takenMetricIds 是新登记的指标短名不能用的编号
|
||||
* (最终指标、常规辅助指标和答案路径的两个保留键)。
|
||||
*/
|
||||
| { kind: "diagnosisProposal"; maxExplanations: number; maxExperiments: number; takenMetricIds: string[] }
|
||||
/** 对自己每条解释(explanations)× 每个合并后的实验(experiments)各给三个数(方法.md 4.3 诊 2)。 */
|
||||
| { kind: "diagnosisPrediction"; explanations: string[]; experiments: string[] }
|
||||
| ({ kind: "diagnosisImplementation" } & DiagnosisItems)
|
||||
| ({ kind: "diagnosisRepair" } & DiagnosisItems)
|
||||
| ({ kind: "diagnosisVerification" } & DiagnosisItems)
|
||||
| ({ kind: "proposal"; maxProposals: number } & FactorRules)
|
||||
| { kind: "firstPrediction"; candidate: string }
|
||||
| { kind: "secondPrediction"; candidate: string }
|
||||
|
||||
@@ -10,6 +10,10 @@ export const STAGES = [
|
||||
"entryDraft", // 0.1 闭门写入口
|
||||
"discussion", // 0.2 讨论,可配置为 1 或 2 次,每次都是这个阶段
|
||||
"claim", // 0.3 认领
|
||||
"diagnosisProposal", // 诊 1 诊断解释与诊断实验
|
||||
"diagnosisPrediction", // 诊 2 诊断预测
|
||||
"diagnosisImplementation", // 诊 4 诊断实现,以及核验不通过后的修复
|
||||
"diagnosisVerification", // 诊 4 诊断核验
|
||||
"proposal", // ② 提案
|
||||
"firstPrediction", // ④ 第一次预测
|
||||
"critique", // ⑤ 挑刺
|
||||
@@ -33,6 +37,11 @@ export const SUBMISSION_KINDS = {
|
||||
entry: { stage: "entryDraft", role: "researcher" },
|
||||
discussion: { stage: "discussion", role: "researcher" },
|
||||
claim: { stage: "claim", role: "researcher" },
|
||||
diagnosisProposal: { stage: "diagnosisProposal", role: "researcher" },
|
||||
diagnosisPrediction: { stage: "diagnosisPrediction", role: "researcher" },
|
||||
diagnosisImplementation: { stage: "diagnosisImplementation", role: "experimenter" },
|
||||
diagnosisRepair: { stage: "diagnosisImplementation", role: "experimenter" },
|
||||
diagnosisVerification: { stage: "diagnosisVerification", role: "verifier" },
|
||||
proposal: { stage: "proposal", role: "researcher" },
|
||||
firstPrediction: { stage: "firstPrediction", role: "researcher" },
|
||||
critique: { stage: "critique", role: "researcher" },
|
||||
|
||||
@@ -503,6 +503,146 @@ export const VerificationSubmission = Type.Object(
|
||||
},
|
||||
);
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// 诊断:解释与实验、预测、实现、核验(方法.md 4.3 诊 1–诊 4、第五部分"诊断"三行)
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
const DiagnosisExplanation = Type.Object(
|
||||
{
|
||||
body: Type.String({
|
||||
...TEXT,
|
||||
description:
|
||||
"Markdown:为什么卡在这里,剩下的空间在哪,什么样的实测会推翻它(写不出推翻它的实测,说明这条解释还不能被检验,先改具体再交)。" +
|
||||
"诊断问的是剩下的空间在哪,不是上一步的功劳怎么分。没有具体判断就直说,写明查了什么、为什么拿不准",
|
||||
}),
|
||||
},
|
||||
CLOSED,
|
||||
);
|
||||
|
||||
const DiagnosisExperiment = Type.Object(
|
||||
{
|
||||
name: Type.String({
|
||||
...TEXT,
|
||||
description: "名称:离开这份清单单独看也读得懂",
|
||||
}),
|
||||
spot: Type.String({
|
||||
...TEXT,
|
||||
description: "动哪一处:在当前基线的哪一处代码或哪一步上动手,只写一处(文件、函数或步骤名),其余照基线保留;想测两处就拆成两个实验",
|
||||
}),
|
||||
body: Type.String({
|
||||
...TEXT,
|
||||
description:
|
||||
"Markdown:在这一处怎么动(换成明显更强的版本、换成另一种做法、拿掉,或放宽某个限制)、" +
|
||||
"做完交出什么能判分的答案、几种可能的结果各排除哪几条解释(有分不开的也写明)。不写自己估的分数,分数在下一步交",
|
||||
}),
|
||||
metrics: Type.Array(HypothesisMetric, {
|
||||
description:
|
||||
"登记假设专用指标:这个实验的代码要额外记录的量,编号只写短名;没有就交空数组。" +
|
||||
"诊断实验里的指标只能选 recordMissing(缺了只记缺失),选 invalidateRun 会被退回",
|
||||
}),
|
||||
},
|
||||
CLOSED,
|
||||
);
|
||||
|
||||
export const DiagnosisProposalSubmission = Type.Object(
|
||||
{
|
||||
explanations: Type.Array(DiagnosisExplanation, {
|
||||
minItems: 1,
|
||||
description: "诊断解释清单,编号由程序分配;条数上限见材料。真有第二个看法再写第二条,各条之间要真的不同;没有具体判断,就写一条直说没有的",
|
||||
}),
|
||||
experiments: Type.Array(DiagnosisExperiment, {
|
||||
minItems: 1,
|
||||
description:
|
||||
"诊断实验清单,个数上限见材料。好的实验是不同解释对它的预测差得远:大家都预测涨不了多少的实验,测了也不改变下一步。" +
|
||||
"每个实验只动一处,用时要放得进分到的那份时间;想不出能把解释分开的实验,就交最能暴露你的解释错在哪的那个",
|
||||
}),
|
||||
},
|
||||
{
|
||||
...CLOSED,
|
||||
description: "对当前基线的诊断:几条解释,和能把它们分开的几个实验。实验正文里不写自己估的分数",
|
||||
},
|
||||
);
|
||||
|
||||
export const DiagnosisPredictionSubmission = Type.Object(
|
||||
{
|
||||
predictions: Type.Array(
|
||||
Type.Object(
|
||||
{
|
||||
explanation: Type.String({ ...Id, description: "你自己的诊断解释编号,照材料里列的写" }),
|
||||
experiment: Type.String({ ...Id, description: "合并后的诊断实验编号,照材料里列的写,别人提的也要给" }),
|
||||
range: Range,
|
||||
},
|
||||
CLOSED,
|
||||
),
|
||||
{
|
||||
description:
|
||||
"你自己的每条诊断解释对每个合并后的实验各给三个数:按这条解释,做了这个实验,分数有一成、一半、九成可能低于多少。" +
|
||||
"不多不少,每一对只写一次;按这条解释本身写,不往折中的范围靠;解释和这个实验没关系就按基线附近写并放宽;拿不准就把 q1 和 q9 拉开",
|
||||
},
|
||||
),
|
||||
body: Type.String({
|
||||
...TEXT,
|
||||
description: "理由,Markdown:这些数是怎么来的,不同解释给的数差在哪里;几条解释给的数差不多,写明它们在这些实验上分不开",
|
||||
}),
|
||||
},
|
||||
CLOSED,
|
||||
);
|
||||
|
||||
/** 诊断实现和修复用同一个定义。代码本身由程序从实验员的工作目录冻结,不作为字段交。 */
|
||||
export const DiagnosisImplementationSubmission = Type.Object(
|
||||
{
|
||||
locations: Type.Array(
|
||||
Type.Object(
|
||||
{
|
||||
experiment: Type.String({ ...Id, description: "诊断实验编号,照材料里列的写" }),
|
||||
locations: LOCATIONS,
|
||||
},
|
||||
CLOSED,
|
||||
),
|
||||
{
|
||||
description: "位置表:选中的每个实验一项,列出本次代码中这个实验实际所在的全部位置,行号以本次提交代码为准",
|
||||
},
|
||||
),
|
||||
concerns: CONCERNS,
|
||||
body: Type.String({
|
||||
...TEXT,
|
||||
description: "实现说明,Markdown:位置表之外需要说明的事;实验原文没规定规模时,写你选了什么、为什么放得进分到的时间。修复时逐条回应核验意见:改在哪里,或以实验原文和代码位置说明为什么不改。没有要补充的就写\"无\"",
|
||||
}),
|
||||
},
|
||||
CLOSED,
|
||||
);
|
||||
|
||||
export const DiagnosisVerificationSubmission = Type.Object(
|
||||
{
|
||||
experiments: Type.Array(
|
||||
Type.Object(
|
||||
{
|
||||
experiment: Type.String({ ...Id, description: "诊断实验编号,照材料里列的写" }),
|
||||
followed: Type.Boolean({
|
||||
description: "照实验写的做法实现了写 true;没找到、做得不一样或材料不足以核清,写 false",
|
||||
}),
|
||||
nothingElse: Type.Boolean({
|
||||
description: "没有动实验写的那一处以外的地方写 true;顺手改了别处(含顺手改进)写 false",
|
||||
}),
|
||||
where: WHERE,
|
||||
},
|
||||
CLOSED,
|
||||
),
|
||||
{
|
||||
description: "选中的每个实验一项,逐个判断,一个实验不通过不影响别的实验",
|
||||
},
|
||||
),
|
||||
body: Type.String({
|
||||
...TEXT,
|
||||
description: "说明,Markdown:逐项结果之外需要说明的事。没有要补充的就写\"无\"",
|
||||
}),
|
||||
},
|
||||
{
|
||||
...CLOSED,
|
||||
description: "只交逐个实验的结果,通过与否由程序推出:任何一项写 false 的实验退回实验员修复,用完修复次数仍不通过的实验从这次诊断里拿掉",
|
||||
},
|
||||
);
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// 解释、消融、收尾(方法.md 4.3 ⑭⑯⑱)
|
||||
// ---------------------------------------------------------------------------
|
||||
@@ -611,6 +751,11 @@ export const SUBMISSION_SCHEMAS = {
|
||||
entry: EntrySubmission,
|
||||
discussion: DiscussionSubmission,
|
||||
claim: ClaimSubmission,
|
||||
diagnosisProposal: DiagnosisProposalSubmission,
|
||||
diagnosisPrediction: DiagnosisPredictionSubmission,
|
||||
diagnosisImplementation: DiagnosisImplementationSubmission,
|
||||
diagnosisRepair: DiagnosisImplementationSubmission,
|
||||
diagnosisVerification: DiagnosisVerificationSubmission,
|
||||
proposal: ProposalSubmission,
|
||||
firstPrediction: PredictionSubmission,
|
||||
critique: CritiqueSubmission,
|
||||
|
||||
Reference in New Issue
Block a user