feat: 诊断(任务书 18 第一块):诊断数学、四种提交、一次运行跑多个实验、账本与材料、flow 诊断各步
消融默认关,关掉时解释阶段一并跳过;选诊断实验用信息价值减成本。 eval/diagnosis/research.json 为第三块配置。vocab 字段说明的出口文字随提示词一起改,见下一次提交的 prompts/ 与同文件改动(本次含在内)。 Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
This commit is contained in:
+59
-1
@@ -3,7 +3,17 @@ import { mkdirSync, mkdtempSync, rmSync, writeFileSync } from "node:fs";
|
||||
import { tmpdir } from "node:os";
|
||||
import { join } from "node:path";
|
||||
import { test, type TestContext } from "node:test";
|
||||
import { checkHunk, checkProtocol, codeHunks, judgeSameChange, type FrozenProtocol, type ProtocolRules } from "../src/jev/index.ts";
|
||||
import {
|
||||
checkExperiment,
|
||||
checkHunk,
|
||||
checkProtocol,
|
||||
codeHunks,
|
||||
judgeSameChange,
|
||||
judgeSameExperiment,
|
||||
type DiagnosisExperiment,
|
||||
type FrozenProtocol,
|
||||
type ProtocolRules,
|
||||
} from "../src/jev/index.ts";
|
||||
import { hash, type Json } from "../src/vocab/index.ts";
|
||||
import { compareVersions, createRepository, saveVersion } from "../src/workspace/index.ts";
|
||||
|
||||
@@ -307,3 +317,51 @@ test("把 workspace 的比较结果切成差异块,每块从 @@ 行起", (t) =
|
||||
assert.equal(hunks[0]?.text, "@@ -1,5 +1,5 @@\n line 1\n-line 2\n+changed 2\n line 3\n line 4\n line 5");
|
||||
assert.equal(hunks[2]?.text, "@@ -0,0 +1 @@\n+x = 1");
|
||||
});
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// 诊断实验的判重和冻结检查(方法.md 4.3 诊 1、诊 2)
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
const EXPERIMENT: DiagnosisExperiment = {
|
||||
name: "拿掉收尾",
|
||||
spot: "solve.py 的收尾修复",
|
||||
body: "拿掉收尾修复,直接交出原始答案",
|
||||
metrics: [{ id: "raw-score", meaning: "收尾之前的分数", unit: "无", direction: "maximize", howMeasured: "收尾之前的分数", missing: "recordMissing" }],
|
||||
};
|
||||
|
||||
test("诊断实验判重:一对一次请求,只给名称、动哪一处、怎么动;阈值暂用判重的 0.3", async (t) => {
|
||||
const sent = fakeJev(t, answerAll(() => 0.31));
|
||||
const other = { name: "强起点", spot: "solve.py 的起点", body: "换成更强的起点" };
|
||||
const j = await judgeSameExperiment("jev-x", EXPERIMENT, other);
|
||||
assert.equal(sent.length, 1);
|
||||
assert.deepEqual(Object.keys(sent[0]!.body.questions), ["same_experiment"]);
|
||||
assert.deepEqual(sent[0]!.body.state.experiment_a, { name: "拿掉收尾", spot: "solve.py 的收尾修复", body: "拿掉收尾修复,直接交出原始答案" });
|
||||
assert.deepEqual(sent[0]!.body.state.experiment_b, other);
|
||||
assert.deepEqual([j.question, j.threshold, j.yes, j.requestId], ["same_experiment", 0.3, true, "req_1"]);
|
||||
});
|
||||
|
||||
test("诊断实验判重:概率等于阈值判否", async (t) => {
|
||||
fakeJev(t, answerAll(() => 0.3));
|
||||
assert.equal((await judgeSameExperiment("jev-x", EXPERIMENT, EXPERIMENT)).yes, false);
|
||||
});
|
||||
|
||||
test("诊断实验冻结检查:三问各一次请求,材料相同,问法是诊断的变体;有一问判是就标出", async (t) => {
|
||||
const sent = fakeJev(t, answerAll((name) => (name === "changes_measurement" ? 0.22 : 0.1)));
|
||||
const result = await checkExperiment("jev-x", EXPERIMENT);
|
||||
assert.deepEqual(
|
||||
result.judgments.map((j) => [j.question, j.threshold, j.yes]),
|
||||
[["self_contradictory", 0.5, false], ["changes_measurement", 0.21, true], ["too_vague", 0.78, false]],
|
||||
);
|
||||
assert.equal(result.flagged, true);
|
||||
assert.deepEqual(sent.map((s) => Object.keys(s.body.questions)), [["self_contradictory"], ["changes_measurement"], ["too_vague"]]);
|
||||
for (const s of sent) assert.deepEqual(s.body.state, sent[0]!.body.state);
|
||||
assert.deepEqual(sent[0]!.body.state.experiment, {
|
||||
name: "拿掉收尾",
|
||||
spot: "solve.py 的收尾修复",
|
||||
body: "拿掉收尾修复,直接交出原始答案",
|
||||
metrics: [{ id: "raw-score", howMeasured: "收尾之前的分数", missing: "recordMissing" }],
|
||||
});
|
||||
// 材料说明和问法都是诊断的变体,哈希和冻结协议同名三问的不同。
|
||||
const protocol = await checkProtocol("jev-x", PROTOCOL);
|
||||
result.judgments.forEach((j, i) => assert.notEqual(j.questionHash, protocol.judgments[i]!.questionHash, j.question));
|
||||
});
|
||||
|
||||
Reference in New Issue
Block a user