feat: 诊断(任务书 18 第一块):诊断数学、四种提交、一次运行跑多个实验、账本与材料、flow 诊断各步

消融默认关,关掉时解释阶段一并跳过;选诊断实验用信息价值减成本。
eval/diagnosis/research.json 为第三块配置。vocab 字段说明的出口文字随提示词一起改,见下一次提交的 prompts/ 与同文件改动(本次含在内)。

Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
This commit is contained in:
2026-10-05 10:36:20 -04:00
parent f39f8410e7
commit 852600df02
69 changed files with 6607 additions and 214 deletions
+59 -1
View File
@@ -3,7 +3,17 @@ import { mkdirSync, mkdtempSync, rmSync, writeFileSync } from "node:fs";
import { tmpdir } from "node:os";
import { join } from "node:path";
import { test, type TestContext } from "node:test";
import { checkHunk, checkProtocol, codeHunks, judgeSameChange, type FrozenProtocol, type ProtocolRules } from "../src/jev/index.ts";
import {
checkExperiment,
checkHunk,
checkProtocol,
codeHunks,
judgeSameChange,
judgeSameExperiment,
type DiagnosisExperiment,
type FrozenProtocol,
type ProtocolRules,
} from "../src/jev/index.ts";
import { hash, type Json } from "../src/vocab/index.ts";
import { compareVersions, createRepository, saveVersion } from "../src/workspace/index.ts";
@@ -307,3 +317,51 @@ test("把 workspace 的比较结果切成差异块,每块从 @@ 行起", (t) =
assert.equal(hunks[0]?.text, "@@ -1,5 +1,5 @@\n line 1\n-line 2\n+changed 2\n line 3\n line 4\n line 5");
assert.equal(hunks[2]?.text, "@@ -0,0 +1 @@\n+x = 1");
});
// ---------------------------------------------------------------------------
// 诊断实验的判重和冻结检查(方法.md 4.3 诊 1、诊 2)
// ---------------------------------------------------------------------------
const EXPERIMENT: DiagnosisExperiment = {
name: "拿掉收尾",
spot: "solve.py 的收尾修复",
body: "拿掉收尾修复,直接交出原始答案",
metrics: [{ id: "raw-score", meaning: "收尾之前的分数", unit: "无", direction: "maximize", howMeasured: "收尾之前的分数", missing: "recordMissing" }],
};
test("诊断实验判重:一对一次请求,只给名称、动哪一处、怎么动;阈值暂用判重的 0.3", async (t) => {
const sent = fakeJev(t, answerAll(() => 0.31));
const other = { name: "强起点", spot: "solve.py 的起点", body: "换成更强的起点" };
const j = await judgeSameExperiment("jev-x", EXPERIMENT, other);
assert.equal(sent.length, 1);
assert.deepEqual(Object.keys(sent[0]!.body.questions), ["same_experiment"]);
assert.deepEqual(sent[0]!.body.state.experiment_a, { name: "拿掉收尾", spot: "solve.py 的收尾修复", body: "拿掉收尾修复,直接交出原始答案" });
assert.deepEqual(sent[0]!.body.state.experiment_b, other);
assert.deepEqual([j.question, j.threshold, j.yes, j.requestId], ["same_experiment", 0.3, true, "req_1"]);
});
test("诊断实验判重:概率等于阈值判否", async (t) => {
fakeJev(t, answerAll(() => 0.3));
assert.equal((await judgeSameExperiment("jev-x", EXPERIMENT, EXPERIMENT)).yes, false);
});
test("诊断实验冻结检查:三问各一次请求,材料相同,问法是诊断的变体;有一问判是就标出", async (t) => {
const sent = fakeJev(t, answerAll((name) => (name === "changes_measurement" ? 0.22 : 0.1)));
const result = await checkExperiment("jev-x", EXPERIMENT);
assert.deepEqual(
result.judgments.map((j) => [j.question, j.threshold, j.yes]),
[["self_contradictory", 0.5, false], ["changes_measurement", 0.21, true], ["too_vague", 0.78, false]],
);
assert.equal(result.flagged, true);
assert.deepEqual(sent.map((s) => Object.keys(s.body.questions)), [["self_contradictory"], ["changes_measurement"], ["too_vague"]]);
for (const s of sent) assert.deepEqual(s.body.state, sent[0]!.body.state);
assert.deepEqual(sent[0]!.body.state.experiment, {
name: "拿掉收尾",
spot: "solve.py 的收尾修复",
body: "拿掉收尾修复,直接交出原始答案",
metrics: [{ id: "raw-score", howMeasured: "收尾之前的分数", missing: "recordMissing" }],
});
// 材料说明和问法都是诊断的变体,哈希和冻结协议同名三问的不同。
const protocol = await checkProtocol("jev-x", PROTOCOL);
result.judgments.forEach((j, i) => assert.notEqual(j.questionHash, protocol.judgments[i]!.questionHash, j.question));
});