feat: 诊断（任务书 18 第一块）：诊断数学、四种提交、一次运行跑多个实验、账本与材料、flow 诊断各步
消融默认关，关掉时解释阶段一并跳过；选诊断实验用信息价值减成本。 eval/diagnosis/research.json 为第三块配置。vocab 字段说明的出口文字随提示词一起改，见下一次提交的 prompts/ 与同文件改动（本次含在内）。 Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
This commit is contained in:
@@ -0,0 +1,185 @@
|
||||
import assert from "node:assert/strict";
|
||||
import { appendFileSync, cpSync, mkdirSync, readFileSync, writeFileSync } from "node:fs";
|
||||
import { join } from "node:path";
|
||||
import { test, type TestContext } from "node:test";
|
||||
import { stepList } from "../src/config/index.ts";
|
||||
import { fullConfig as checkedFull } from "../src/flow/frozen.ts";
|
||||
import { runPaths, workspaceRepo } from "../src/flow/paths.ts";
|
||||
import { groupSameExperiment, selectDiagnosisExperiments } from "../src/flow/steps/diagnosis.ts";
|
||||
import { prepareSubmission, taskArtifact, taskCode } from "../src/flow/task-code.ts";
|
||||
import type { Study } from "../src/flow/study.ts";
|
||||
import { Ledger } from "../src/ledger/index.ts";
|
||||
import { runDiagnosisRule, type DiagnosisRuleInput } from "../src/rules/index.ts";
|
||||
import type { ForwardCode } from "../src/submissions/index.ts";
|
||||
import { checkTaskFacts, diagnosisUnversionedNames, loadTask, placeBaselineArtifact } from "../src/tasks/index.ts";
|
||||
import type { JevJudgment } from "../src/vocab/index.ts";
|
||||
import { compareVersions, createRepository, saveVersion } from "../src/workspace/index.ts";
|
||||
import { fullConfig } from "./config-helpers.ts";
|
||||
import { tempDir } from "./ledger-helpers.ts";
|
||||
import { judged, r, Writer } from "./materials-helpers.ts";
|
||||
|
||||
// 诊断里程序自己做的两步（任务书 18 第 1.6 节）：判重合并的分组、选诊断实验；加上诊断实现、修复、核验的取材和存版本接线。判重合并的 Jev 问法由 jev 的测试管；
|
||||
// 诊断运行和结算要真实 Docker，在 flow-diagnosis-docker.test.ts。预期值照条文手拼，输出用 rules 的函数当对照
|
||||
// （同选格测试的写法，rules 自己有独立预期值的测试）。
|
||||
|
||||
const facts = checkTaskFacts(JSON.parse(readFileSync(join(import.meta.dirname, "..", "tasks", "circle_packing", "task.json"), "utf8")));
|
||||
|
||||
const yes = (probability: number): JevJudgment => ({ question: "diagnosisDedup", probability, threshold: 0.3, yes: probability > 0.3 }) as unknown as JevJudgment;
|
||||
const item = (author: string, index: number) => ({ author, index, label: `${author}.e${index + 1}` });
|
||||
|
||||
test("诊断实验判重合并：判为相同的连成一组，编号 d1… 按组里排最前的实验排，组内按作者、第几个排", () => {
|
||||
const items = [item("researcher-1", 0), item("researcher-1", 1), item("researcher-2", 0), item("researcher-3", 0)];
|
||||
const pairs = [
|
||||
{ a: "researcher-1.e1", b: "researcher-2.e1", judgment: yes(0.9) },
|
||||
{ a: "researcher-1.e2", b: "researcher-3.e1", judgment: yes(0.1) },
|
||||
{ a: "researcher-2.e1", b: "researcher-3.e1", judgment: yes(0.8) },
|
||||
];
|
||||
// researcher-1.e1、researcher-2.e1、researcher-3.e1 连成一组（A=B，B=C）；researcher-1.e2 自成一个。
|
||||
assert.deepEqual(groupSameExperiment(items, pairs), {
|
||||
experiments: [
|
||||
{ experiment: "d1", sources: [{ author: "researcher-1", index: 0 }, { author: "researcher-2", index: 0 }, { author: "researcher-3", index: 0 }] },
|
||||
{ experiment: "d2", sources: [{ author: "researcher-1", index: 1 }] },
|
||||
],
|
||||
});
|
||||
});
|
||||
|
||||
test("诊断实验判重合并：没有一对判相同时，每个实验各成一个", () => {
|
||||
const items = [item("researcher-1", 0), item("researcher-2", 0)];
|
||||
const pairs = [{ a: "researcher-1.e1", b: "researcher-2.e1", judgment: yes(0.1) }];
|
||||
assert.deepEqual(groupSameExperiment(items, pairs).experiments.map((e) => e.experiment), ["d1", "d2"]);
|
||||
});
|
||||
|
||||
/** 基线 0.96；a1、a2 各一条诊断解释，对合并后的 d1、d2 各给三个数；余额：13 次里基线用了 1 次。 */
|
||||
function study(t: TestContext, ranges: Record<string, ReturnType<typeof r>[]>): { s: Study; input: DiagnosisRuleInput } {
|
||||
const ledger = Ledger.open(join(tempDir(t), "ledger.db"));
|
||||
t.after(() => ledger.close());
|
||||
const w = new Writer(ledger);
|
||||
ledger.append("code", { object: "base", version: "v-base", locations: null });
|
||||
w.run("base-1", "baseline", 0, null, judged(0.96, "main"));
|
||||
w.open("r1-diagnosisProposal", "diagnosisProposal", 1, ["a1", "a2"]);
|
||||
const experiment = (name: string) => ({ name, spot: `${name}的位置`, body: `${name}的做法`, metrics: [] });
|
||||
for (const [author, name] of [["a1", "实验甲"], ["a2", "实验乙"]] as const) {
|
||||
w.sub("r1-diagnosisProposal", author, "diagnosisProposal", null, null, { explanations: [{ body: `${author}的解释` }], experiments: [experiment(name)] });
|
||||
}
|
||||
w.close("r1-diagnosisProposal");
|
||||
w.decision("diagnosisDedup", 1, null, {
|
||||
experiments: [{ experiment: "d1", sources: [{ author: "a1", index: 0 }] }, { experiment: "d2", sources: [{ author: "a2", index: 0 }] }],
|
||||
});
|
||||
w.open("r1-diagnosisPrediction", "diagnosisPrediction", 1, ["a1", "a2"]);
|
||||
for (const author of ["a1", "a2"]) {
|
||||
const mine = ranges[author]!;
|
||||
w.sub("r1-diagnosisPrediction", author, "diagnosisPrediction", null, null, {
|
||||
predictions: ["d1", "d2"].map((d, i) => ({ explanation: `r1-${author}-dx1`, experiment: d, range: mine[i]! })),
|
||||
body: "理由",
|
||||
});
|
||||
}
|
||||
w.close("r1-diagnosisPrediction");
|
||||
const config = fullConfig();
|
||||
if (config.flow.preset !== "full") throw new Error("测试配置应是完整流程");
|
||||
config.flow = { ...config.flow, diagnosis: { enabled: true, perResearcher: 2, maxExperiments: 4 }, ablation: { enabled: false } };
|
||||
const frozen = { config: checkedFull(config), steps: stepList(config), facts, prompts: null, image: "", seed: 1 };
|
||||
const input: DiagnosisRuleInput = {
|
||||
baseline: 0.96,
|
||||
direction: "maximize",
|
||||
lambda: 0.1,
|
||||
experiments: ["d1", "d2"],
|
||||
explanations: ["a1", "a2"].map((author) => ({
|
||||
id: `r1-${author}-dx1`,
|
||||
author,
|
||||
experiments: ["d1", "d2"].map((experiment, i) => ({ experiment, range: ranges[author]![i]! })),
|
||||
})),
|
||||
balance: 12,
|
||||
batchCap: 2,
|
||||
maxExperiments: 4,
|
||||
u: 0.5,
|
||||
decay: 0.5,
|
||||
};
|
||||
return { s: { ledger, frozen } as unknown as Study, input };
|
||||
}
|
||||
|
||||
test("选诊断实验：输入照 DiagnosisRuleInput 拼（基线、每条解释对每个实验的三个数、余额从本轮之前的运行算），规则取步骤清单，种子 null", (t) => {
|
||||
// 两人对 d1 预测相反（一个说涨到 1.3，一个说跌到 0.9），对 d2 都说涨不了多少。
|
||||
const { s, input } = study(t, {
|
||||
a1: [r(1.2, 1.3, 1.4), r(0.95, 0.97, 0.99)],
|
||||
a2: [r(0.8, 0.9, 1.0), r(0.95, 0.97, 0.99)],
|
||||
});
|
||||
selectDiagnosisExperiments(s, 1);
|
||||
const record = s.ledger.records().find((x) => x.kind === "decision" && x.content.decision === "selectDiagnosis")!;
|
||||
const { rule: used, ...output } = runDiagnosisRule("valueMinusCost", input);
|
||||
assert.deepEqual(record.content, { decision: "selectDiagnosis", round: 1, rule: "valueMinusCost", ruleVersion: used.version, params: output.params, seed: null, input, output });
|
||||
// 分歧大的实验信息价值为正：本轮做诊断，d1 在选中的组合里。
|
||||
assert.equal(output.diagnose, true);
|
||||
assert.ok(output.chosen.includes("d1"));
|
||||
});
|
||||
|
||||
/** 诊断实现的代码接线：真实裸仓库里存了基线；工作目录是基线副本加一处改动，再放进只读的原始产物。 */
|
||||
function codeStudy(t: TestContext): { s: Study; work: string; baseline: string } {
|
||||
const dir = tempDir(t);
|
||||
const ledger = Ledger.open(join(dir, "ledger.db"));
|
||||
t.after(() => ledger.close());
|
||||
const task = loadTask(join(import.meta.dirname, "..", "tasks", "circle_packing"));
|
||||
const repo = workspaceRepo(dir);
|
||||
createRepository(repo);
|
||||
const names = diagnosisUnversionedNames(task);
|
||||
const saved = saveVersion(repo, task.baselineDir, names);
|
||||
if (!saved.saved) throw new Error("基线存不成版本");
|
||||
const w = new Writer(ledger);
|
||||
ledger.append("code", { object: "base", version: saved.version, locations: null });
|
||||
w.run("base-1", "baseline", 0, null, judged(0.96, "main"));
|
||||
const baseRun = runPaths(dir, "base-1").run;
|
||||
mkdirSync(baseRun, { recursive: true });
|
||||
writeFileSync(join(baseRun, "solution.json"), "{}");
|
||||
w.open("r1-diagnosisProposal", "diagnosisProposal", 1, ["a1"]);
|
||||
const metric = { id: "note", meaning: "记录", unit: "无", direction: "maximize" as const, howMeasured: "数一数", missing: "recordMissing" as const };
|
||||
w.sub("r1-diagnosisProposal", "a1", "diagnosisProposal", null, null, { explanations: [{ body: "解释" }], experiments: [{ name: "实验甲", spot: "第一步", body: "换个做法", metrics: [metric] }] });
|
||||
w.close("r1-diagnosisProposal");
|
||||
w.decision("diagnosisDedup", 1, null, { experiments: [{ experiment: "d1", sources: [{ author: "a1", index: 0 }] }] });
|
||||
w.decision("selectDiagnosis", 1, null, { diagnose: true, chosen: ["d1"] });
|
||||
const work = join(dir, "work");
|
||||
cpSync(task.baselineDir, work, { recursive: true });
|
||||
appendFileSync(join(work, "initial.py"), "\n\ndef run_experiment(experiment_id, baseline_dir):\n return [], [], 0.0\n");
|
||||
placeBaselineArtifact(task, baseRun, work);
|
||||
const frozen = { config: checkedFull(fullConfig()), steps: stepList(fullConfig()), facts: task.facts, prompts: null, image: "", seed: 1 };
|
||||
return { s: { dir, task, ledger, frozen } as unknown as Study, work, baseline: saved.version };
|
||||
}
|
||||
|
||||
const dxTask = (kind: "diagnosisImplementation" | "diagnosisRepair" | "diagnosisVerification") =>
|
||||
({ actor: "x", kind, object: "r1-diagnosis", stageId: `r1-${kind}-1` });
|
||||
|
||||
test("诊断实现的接线：存版本时跳过只读的原始产物目录；Jev 对照选中实验的原文和全名指标；工作目录放进原始产物", (t) => {
|
||||
const { s, work, baseline } = codeStudy(t);
|
||||
const w = new Writer(s.ledger);
|
||||
w.open("r1-diagnosisImplementation-1", "diagnosisImplementation", 1, ["x"]);
|
||||
const content = { locations: [{ experiment: "d1", locations: ["initial.py:90-92"] }], concerns: "无", body: "无" };
|
||||
const { known, code } = prepareSubmission(s, dxTask("diagnosisImplementation"), content, 0, work);
|
||||
const saved = (known as { code: ForwardCode }).code;
|
||||
if (!saved.saved) throw new Error("应存成版本");
|
||||
assert.deepEqual(saved.protectedFiles, []);
|
||||
assert.ok(saved.hunks.length > 0);
|
||||
assert.ok(saved.hunks.every((h) => h.path === "initial.py"));
|
||||
assert.match(saved.protocol.body, /### d1 实验甲\n\n动哪一处：第一步\n\n换个做法/);
|
||||
assert.deepEqual(saved.protocol.metrics.map((m) => m.id), ["d1.note"]);
|
||||
assert.deepEqual(code, { object: "r1-diagnosis", version: code!.version, locations: content.locations });
|
||||
// 工作目录里放着的原始产物没有进版本：当前基线到诊断程序的改动里只有 initial.py。
|
||||
const changes = compareVersions(workspaceRepo(s.dir), baseline, code!.version);
|
||||
assert.deepEqual(changes.files.map((f) => f.path), ["initial.py"]);
|
||||
});
|
||||
|
||||
test("诊断实现、修复、核验取材：实现从当前基线出发，修复从上次提交的诊断程序出发，核验和修复的差异都是当前基线到诊断程序；原始产物只给实现和修复", (t) => {
|
||||
const { s, work, baseline } = codeStudy(t);
|
||||
const w = new Writer(s.ledger);
|
||||
const content = { locations: [{ experiment: "d1", locations: ["initial.py:90-92"] }], concerns: "无", body: "无" };
|
||||
w.open("r1-diagnosisImplementation-1", "diagnosisImplementation", 1, ["x"]);
|
||||
w.open("r1-diagnosisRepair-1", "diagnosisImplementation", 1, ["x"]);
|
||||
w.open("r1-diagnosisVerification-1", "diagnosisVerification", 1, ["v"]);
|
||||
assert.deepEqual(taskCode(s, dxTask("diagnosisImplementation")), { changes: null, startVersion: baseline });
|
||||
const { code } = prepareSubmission(s, dxTask("diagnosisImplementation"), content, 0, work);
|
||||
s.ledger.append("code", code!);
|
||||
const changes = compareVersions(workspaceRepo(s.dir), baseline, code!.version);
|
||||
assert.deepEqual(taskCode(s, dxTask("diagnosisRepair")), { changes, startVersion: code!.version });
|
||||
assert.deepEqual(taskCode(s, dxTask("diagnosisVerification")), { changes, startVersion: null });
|
||||
const baseRun = runPaths(s.dir, "base-1").run;
|
||||
assert.equal(taskArtifact(s, dxTask("diagnosisImplementation")), baseRun);
|
||||
assert.equal(taskArtifact(s, dxTask("diagnosisRepair")), baseRun);
|
||||
assert.equal(taskArtifact(s, dxTask("diagnosisVerification")), null);
|
||||
});
|
||||
Reference in New Issue
Block a user