feat: 诊断（任务书 18 第一块）：诊断数学、四种提交、一次运行跑多个实验、账本与材料、flow 诊断各步

消融默认关，关掉时解释阶段一并跳过；选诊断实验用信息价值减成本。 eval/diagnosis/research.json 为第三块配置。vocab 字段说明的出口文字随提示词一起改，见下一次提交的 prompts/ 与同文件改动（本次含在内）。 Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
2026-10-05 10:36:20 -04:00
parent f39f8410e7
commit 852600df02
69 changed files with 6607 additions and 214 deletions
@@ -2,6 +2,7 @@ import assert from "node:assert/strict";
 import { test, type TestContext } from "node:test";
 import {
  ablationOutcome,
+  diagnosisOutcome,
  protocolBody,
  receive,
  verificationOutcome,
@@ -442,3 +443,138 @@ test("消融实现的代码存不成版本：退回", async () => {
  });
  assert.match(reason, /代码存不成版本：bin\/x（符号链接/);
 });
+
+// ---------------------------------------------------------------------------
+// 诊断（方法.md 4.3 诊 1–诊 4）
+// ---------------------------------------------------------------------------
+
+const DX_EXPERIMENT = (name: string, missing: "recordMissing" | "invalidateRun" = "recordMissing", ids: string[] = []) => ({
+  name,
+  spot: "solve.py 的起点生成",
+  body: "换成更强的起点，交出答案",
+  metrics: ids.map((id) => ({ ...metric(id), missing })),
+});
+const DX_PROPOSAL = (over: Partial<Submission<"diagnosisProposal">> = {}): Submission<"diagnosisProposal"> => ({
+  explanations: [{ body: "卡在起点" }, { body: "卡在收尾" }],
+  experiments: [DX_EXPERIMENT("强起点"), DX_EXPERIMENT("拿掉收尾", "recordMissing", ["raw-score"])],
+  ...over,
+});
+const DX_LIST: ChecklistOf<"diagnosisProposal"> = { kind: "diagnosisProposal", maxExplanations: 2, maxExperiments: 2, takenMetricIds: ["sum-of-radii", "answer-path"] };
+const dxProposal = (content: Submission<"diagnosisProposal">): Incoming => ({ kind: "diagnosisProposal", content, checklist: DX_LIST, known: null, jevModel: MODEL });
+
+test("诊断解释与实验：解释 1 至 M 条、实验 1 至 A 个；超过就退回", async () => {
+  const many = DX_PROPOSAL({ explanations: [{ body: "a" }, { body: "b" }, { body: "c" }], experiments: [DX_EXPERIMENT("1"), DX_EXPERIMENT("2"), DX_EXPERIMENT("3")] });
+  const reason = await rejected(dxProposal(many));
+  assert.match(reason, /交了 3 条诊断解释，最多 2 条/);
+  assert.match(reason, /交了 3 个诊断实验，最多 2 个/);
+});
+
+test("诊断解释与实验：登记的指标只能是 recordMissing；同一个实验里不重名、不占用已有编号", async () => {
+  const reason = await rejected(
+    dxProposal(
+      DX_PROPOSAL({
+        experiments: [
+          DX_EXPERIMENT("甲", "invalidateRun", ["raw-score"]),
+          { ...DX_EXPERIMENT("乙"), metrics: [metric("a"), metric("a"), metric("answer-path")] },
+        ],
+      }),
+    ),
+  );
+  assert.match(reason, /第 1 个实验的指标 raw-score 选了 invalidateRun：诊断实验里的指标只能选 recordMissing/);
+  assert.match(reason, /第 2 个实验的 metrics 里编号 a 写了不止一次/);
+  assert.match(reason, /第 2 个实验的指标编号 answer-path 已被占用/);
+});
+
+test("诊断解释与实验：对每个实验问冻结三问；两个实验用同一个指标短名没问题（冻结时各带自己实验的前缀）", async (t) => {
+  fakeJev(t, () => false);
+  const v = await receive(dxProposal(DX_PROPOSAL({ experiments: [DX_EXPERIMENT("甲", "recordMissing", ["x"]), DX_EXPERIMENT("乙", "recordMissing", ["x"])] })));
+  assert.equal(v.accepted, true);
+  assert.equal(v.jev!.length, 6, "两个实验各三问");
+  assert.deepEqual(v.jev!.map((j) => j.question), ["self_contradictory", "changes_measurement", "too_vague", "self_contradictory", "changes_measurement", "too_vague"]);
+});
+
+test("诊断解释与实验：Jev 判是就退回，原因写明第几个实验和哪一问", async (t) => {
+  fakeJev(t, (q) => q === "too_vague");
+  const v = await receive(dxProposal(DX_PROPOSAL()));
+  assert.equal(v.accepted, false);
+  const reason = (v as Extract<Verdict, { accepted: false }>).reason;
+  assert.match(reason, /Jev 判出下面几处有问题/);
+  assert.match(reason, /- 第 1 个诊断实验（强起点）：too_vague 概率 0\.99/);
+  assert.match(reason, /- 第 2 个诊断实验（拿掉收尾）：too_vague 概率 0\.99/);
+});
+
+const DX_PRED_LIST: ChecklistOf<"diagnosisPrediction"> = { kind: "diagnosisPrediction", explanations: ["r1-dx1", "r1-dx2"], experiments: ["d1", "d2"] };
+const dxPred = (predictions: Array<[string, string, ReturnType<typeof r>]>): Incoming => ({
+  kind: "diagnosisPrediction",
+  content: { predictions: predictions.map(([explanation, experiment, range]) => ({ explanation, experiment, range })), body: "理由" },
+  checklist: DX_PRED_LIST,
+  known: null,
+  jevModel: MODEL,
+});
+
+test("诊断预测：自己每条解释 × 每个合并后的实验，不多不少；收下时不问 Jev", async () => {
+  const all: Array<[string, string, ReturnType<typeof r>]> = [
+    ["r1-dx1", "d1", r(1, 1.1, 1.2)], ["r1-dx1", "d2", r(1, 1.1, 1.2)], ["r1-dx2", "d1", r(1, 1.1, 1.2)], ["r1-dx2", "d2", r(1, 1.1, 1.2)],
+  ];
+  assert.deepEqual(await receive(dxPred(all)), { accepted: true, jev: null, protocol: null });
+  const reason = await rejected(dxPred([all[0]!, all[0]!, all[1]!, ["r1-dx3", "d1", r(1, 1.1, 1.2)]]));
+  assert.match(reason, /predictions 的（解释:实验）应当是 r1-dx1:d1、r1-dx1:d2、r1-dx2:d1、r1-dx2:d2/);
+  assert.match(reason, /你写的 r1-dx3:d1 不在里面/);
+  assert.match(reason, /还缺 r1-dx2:d1、r1-dx2:d2/);
+  assert.match(reason, /r1-dx1:d1 写了不止一次/);
+});
+
+test("诊断预测：三个数要严格递增", async () => {
+  const reason = await rejected(dxPred([["r1-dx1", "d1", r(1, 1, 1.2)], ["r1-dx1", "d2", r(1, 1.1, 1.2)], ["r1-dx2", "d1", r(1, 1.1, 1.2)], ["r1-dx2", "d2", r(1.3, 1.2, 1.1)]]));
+  assert.match(reason, /r1-dx1:d1 的 range 的三个数要严格递增 q1 < q5 < q9，你写的是 1 \/ 1 \/ 1\.2/);
+  assert.match(reason, /r1-dx2:d2 的 range 的三个数要严格递增/);
+});
+
+const DX_CODE_BODY = { locations: [{ experiment: "d1", locations: ["solve.py:3-9"] }], concerns: "无", body: "无" };
+type DxCode = Extract<Incoming, { kind: "diagnosisImplementation" }>["known"]["code"];
+const dxCode = (kind: "diagnosisImplementation" | "diagnosisRepair", code: DxCode): Incoming =>
+  kind === "diagnosisImplementation"
+    ? { kind, content: DX_CODE_BODY, checklist: { kind, experiments: ["d1"] }, known: { code }, jevModel: MODEL }
+    : { kind, content: DX_CODE_BODY, checklist: { kind, experiments: ["d1"] }, known: { code }, jevModel: MODEL };
+
+test("诊断实现、修复：改了不许改的文件、代码存不了就退回；有差异块时每块问一次五问", async (t) => {
+  for (const kind of ["diagnosisImplementation", "diagnosisRepair"] as const) {
+    assert.match(await rejected(dxCode(kind, { ...SAVED, protectedFiles: ["validate.py"] })), /改了不许改的文件：validate\.py/);
+    assert.match(await rejected(dxCode(kind, { saved: false, path: "bin/x", reason: "符号链接" })), /代码存不成版本：bin\/x（符号链接）/);
+  }
+  fakeJev(t, () => false);
+  const hunks = [
+    { path: "solve.py", change: "modified" as const, text: "@@ -1 +1 @@\n-a\n+b" },
+    { path: "solve.py", change: "modified" as const, text: "@@ -9 +9 @@\n-c\n+d" },
+  ];
+  const v = await receive(dxCode("diagnosisRepair", { ...SAVED, hunks }));
+  assert.equal(v.accepted, true);
+  assert.equal(v.jev!.length, 10, "两块各五问");
+});
+
+test("诊断核验：逐个实验的结果覆盖全部选中的实验", async () => {
+  const verification = (ids: string[]): Incoming => ({
+    kind: "diagnosisVerification",
+    content: { experiments: ids.map((experiment) => ({ experiment, followed: true, nothingElse: true, where: "solve.py:3" })), body: "无" },
+    checklist: { kind: "diagnosisVerification", experiments: ["d1", "d2"] },
+    known: null,
+    jevModel: MODEL,
+  });
+  assert.deepEqual(await receive(verification(["d2", "d1"])), { accepted: true, jev: null, protocol: null });
+  const reason = await rejected(verification(["d1", "d3"]));
+  assert.match(reason, /experiments 的实验应当是 d1、d2；你写的 d3 不在里面；还缺 d2/);
+});
+
+test("诊断核验通过没有：逐个实验判，一个不通过不影响别的实验", () => {
+  const v: Submission<"diagnosisVerification"> = {
+    experiments: [
+      { experiment: "d1", followed: true, nothingElse: true, where: "a" },
+      { experiment: "d2", followed: false, nothingElse: true, where: "没找到" },
+      { experiment: "d3", followed: true, nothingElse: false, where: "顺手改了别处" },
+      { experiment: "d4", followed: false, nothingElse: false, where: "都不对" },
+    ],
+    body: "无",
+  };
+  assert.deepEqual(diagnosisOutcome(v), { passed: false, failing: ["d2", "d3", "d4"], notFollowed: ["d2", "d4"], touchedElse: ["d3", "d4"] });
+  assert.deepEqual(diagnosisOutcome({ experiments: [v.experiments[0]!], body: "无" }), { passed: true, failing: [], notFollowed: [], touchedElse: [] });
+});