iomgaa
852600df02
消融默认关，关掉时解释阶段一并跳过；选诊断实验用信息价值减成本。 eval/diagnosis/research.json 为第三块配置。vocab 字段说明的出口文字随提示词一起改，见下一次提交的 prompts/ 与同文件改动（本次含在内）。 Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
581 lines
36 KiB
TypeScript
581 lines
36 KiB
TypeScript
import assert from "node:assert/strict";
|
||
import { test, type TestContext } from "node:test";
|
||
import {
|
||
ablationOutcome,
|
||
diagnosisOutcome,
|
||
protocolBody,
|
||
receive,
|
||
verificationOutcome,
|
||
type Incoming,
|
||
type OriginalProposal,
|
||
type Verdict,
|
||
} from "../src/submissions/index.ts";
|
||
import type { ChecklistOf, Submission } from "../src/vocab/index.ts";
|
||
|
||
// submissions 的不花钱测试（任务书 10 第六部分）。收下的例子用不问 Jev 的种类，或者在程序检查那一步就退回；
|
||
// 退回原因的预期文字按任务书第四部分的写法手写。走到真实 Jev 的路径在 tests/jev-real.ts。
|
||
// 只有"差异块退回文字"一条用假 Jev：把全局 fetch 换成假的，在测试自己的环境里造回答（同 tests/jev.test.ts）。
|
||
|
||
const MODEL = "jev-test";
|
||
const r = (q1: number, q5: number, q9: number) => ({ q1, q5, q9 });
|
||
const factor = (name: string, continues: string | null) => ({ name, method: `${name}的做法`, keySettings: "无", continues });
|
||
const metric = (id: string) => ({ id, meaning: "次数", unit: "次", direction: "maximize" as const, howMeasured: "数", missing: "recordMissing" as const });
|
||
|
||
// 能引用的是冻结后的全名"候选编号.短名"；已占用的只有任务的指标和两个保留键（任务书 11 第一部分）。
|
||
const RULES = { factorLimit: 2, continuable: ["r1-c1-p1", "r1-c1-p2"], reusableMetrics: ["r1-c1.restart-count", "r1-c2.restart-count"], takenMetricIds: ["sum-of-radii", "wall-seconds", "answer-path", "safeguard-triggers"] };
|
||
|
||
const PROPOSAL: Submission<"proposal"> = {
|
||
factors: [factor("多起点", null), factor("扰动", "r1-c1-p2")],
|
||
safeguards: ["guard"],
|
||
metrics: { defined: [metric("kick-count")], reused: ["r1-c1.restart-count"] },
|
||
authorEstimate: r(1.1, 1.2, 1.3),
|
||
body: "提案正文",
|
||
};
|
||
const PROPOSAL_LIST: ChecklistOf<"proposal"> = { kind: "proposal", maxProposals: 2, ...RULES };
|
||
|
||
/** 退回，并交回原因文字；顺带确认没问 Jev。 */
|
||
async function rejected(inc: Incoming): Promise<string> {
|
||
const v = await receive(inc);
|
||
assert.equal(v.accepted, false, "应当退回");
|
||
assert.equal(v.jev, null, "程序检查就退回，不问 Jev");
|
||
return (v as Extract<Verdict, { accepted: false }>).reason;
|
||
}
|
||
|
||
const proposal = (content: Submission<"proposal">, accepted: number): Incoming => ({ kind: "proposal", content, checklist: PROPOSAL_LIST, known: { accepted }, jevModel: MODEL });
|
||
|
||
// ---------------------------------------------------------------------------
|
||
// 收下
|
||
// ---------------------------------------------------------------------------
|
||
|
||
test("都对的提案、预测、挑刺、志愿、解释、消融核验收下，不问 Jev，jev 与 protocol 为 null", async () => {
|
||
const cases: Incoming[] = [
|
||
proposal(PROPOSAL, 1),
|
||
{ kind: "firstPrediction", content: { range: r(1, 1.1, 1.2), body: "理由" }, checklist: { kind: "firstPrediction", candidate: "r2-c1" }, known: null, jevModel: MODEL },
|
||
{ kind: "critique", content: { target: "whole", body: "挑刺" }, checklist: { kind: "critique", proposal: "r2-a1-1", factors: ["p1"] }, known: null, jevModel: MODEL },
|
||
{ kind: "claim", content: { choices: ["e-a1-1", "e-a2-1"] }, checklist: { kind: "claim", entries: ["e-a1-1", "e-a2-1", "e-a2-2"], minChoices: 2 }, known: null, jevModel: MODEL },
|
||
{
|
||
kind: "explanation",
|
||
content: { explanations: [{ body: "机制", cells: [{ factor: "p2", range: r(1, 1.1, 1.2) }, { factor: "p1", range: r(0.9, 1, 1.1) }] }] },
|
||
checklist: { kind: "explanation", cells: ["p1", "p2"], maxExplanations: 2 },
|
||
known: null,
|
||
jevModel: MODEL,
|
||
},
|
||
{
|
||
kind: "ablationVerification",
|
||
content: { factors: [{ factor: "p1", off: true, where: "a.py:1" }, { factor: "p2", off: false, where: "a.py:9" }] },
|
||
checklist: { kind: "ablationVerification", cell: "r1-c1-p1", factors: ["p1", "p2"] },
|
||
known: null,
|
||
jevModel: MODEL,
|
||
},
|
||
{ kind: "wrapUp", content: { body: "收尾" }, checklist: { kind: "wrapUp" }, known: null, jevModel: MODEL },
|
||
];
|
||
for (const inc of cases) assert.deepEqual(await receive(inc), { accepted: true, jev: null, protocol: null }, inc.kind);
|
||
});
|
||
|
||
test("消融实现：跑得起来、没改不许改的文件，收下；交跑不起来时不看代码", async () => {
|
||
const checklist: ChecklistOf<"ablationImplementation"> = { kind: "ablationImplementation", cell: "r1-c1-p1", off: "p1", factors: ["p1", "p2", "p3"] };
|
||
const ok = await receive({ kind: "ablationImplementation", content: { alsoOff: ["p3"], cannotRun: null, concerns: "无" }, checklist, known: { code: { saved: true, protectedFiles: [] } }, jevModel: MODEL });
|
||
assert.deepEqual(ok, { accepted: true, jev: null, protocol: null });
|
||
const cannot = await receive({ kind: "ablationImplementation", content: { alsoOff: [], cannotRun: "关了 p1 程序起不来", concerns: "无" }, checklist, known: { code: null }, jevModel: MODEL });
|
||
assert.deepEqual(cannot, { accepted: true, jev: null, protocol: null });
|
||
});
|
||
|
||
test("实现：没有差异块（例如只改了可执行位）时 Jev 看不到内容，不问，jev 为 null", async () => {
|
||
const v = await receive(implementation(SAVED, TABLE));
|
||
assert.deepEqual(v, { accepted: true, jev: null, protocol: null });
|
||
});
|
||
|
||
// ---------------------------------------------------------------------------
|
||
// 程序检查：每条规则一个该退回的例子
|
||
// ---------------------------------------------------------------------------
|
||
|
||
test("规则 1：三个数要严格递增（作者自评、两次预测、解释每格）", async () => {
|
||
assert.match(await rejected(proposal({ ...PROPOSAL, authorEstimate: r(1.2, 1.2, 1.3) }, 0)), /authorEstimate 的三个数要严格递增 q1 < q5 < q9，你写的是 1.2 \/ 1.2 \/ 1.3/);
|
||
const reason = await rejected({ kind: "secondPrediction", content: { range: r(1.3, 1.2, 1.1), body: "理由" }, checklist: { kind: "secondPrediction", candidate: "r1-c1" }, known: null, jevModel: MODEL });
|
||
assert.equal(reason, "这份提交没有收下，要改的地方：\n- range 的三个数要严格递增 q1 < q5 < q9，你写的是 1.3 / 1.2 / 1.1");
|
||
const expl = await rejected({
|
||
kind: "explanation",
|
||
content: { explanations: [{ body: "机制", cells: [{ factor: "p1", range: r(1, 1, 1) }] }] },
|
||
checklist: { kind: "explanation", cells: ["p1"], maxExplanations: 1 },
|
||
known: null,
|
||
jevModel: MODEL,
|
||
});
|
||
assert.match(expl, /第 1 条解释 p1 格的 range 的三个数要严格递增/);
|
||
});
|
||
|
||
test("规则 2：因子数超过上限退回", async () => {
|
||
const reason = await rejected(proposal({ ...PROPOSAL, factors: [factor("a", null), factor("b", null), factor("c", null)] }, 0));
|
||
assert.match(reason, /factors 有 3 个因子，超过上限 2 个/);
|
||
});
|
||
|
||
test("规则 3：两个因子不能延续同一个；延续的编号要在可延续清单里", async () => {
|
||
assert.match(await rejected(proposal({ ...PROPOSAL, factors: [factor("a", "r1-c1-p1"), factor("b", "r1-c1-p1")] }, 0)), /有两个以上因子的 continues 都写了 r1-c1-p1/);
|
||
const reason = await rejected(proposal({ ...PROPOSAL, factors: [factor("a", "r1-c9-p1")] }, 0));
|
||
assert.match(reason, /continues 只能写 r1-c1-p1、r1-c1-p2；你写的 r1-c9-p1 不在里面；新因子写 null/);
|
||
});
|
||
|
||
test("规则 4：保障短名不能叫 main、不能写成因子编号、不能重复", async () => {
|
||
const reason = await rejected(proposal({ ...PROPOSAL, safeguards: ["main", "p3", "guard", "guard"] }, 0));
|
||
assert.match(reason, /保障短名 main 不能用/);
|
||
assert.match(reason, /保障短名 p3 不能用/);
|
||
assert.match(reason, /safeguards 里 guard 写了不止一次/);
|
||
});
|
||
|
||
test("规则 5：每条解释都要交 cells 并恰好覆盖材料列出的格（用户 2026-10-03 定：只按全开实现）", async () => {
|
||
const checklist: ChecklistOf<"explanation"> = { kind: "explanation", cells: ["p1", "p2"], maxExplanations: 2 };
|
||
const missing = await rejected({ kind: "explanation", content: { explanations: [{ body: "机制" }] }, checklist, known: null, jevModel: MODEL });
|
||
assert.match(missing, /第 1 条解释没交 cells：每条解释都要对 p1、p2 每一格给三个数/);
|
||
const wrong = await rejected({ kind: "explanation", content: { explanations: [{ body: "机制", cells: [{ factor: "p1", range: r(1, 2, 3) }, { factor: "p3", range: r(1, 2, 3) }] }] }, checklist, known: null, jevModel: MODEL });
|
||
assert.match(wrong, /第 1 条解释 cells 的格应当是 p1、p2；你写的 p3 不在里面；还缺 p2/);
|
||
});
|
||
|
||
test("规则 6：解释最多 M 条", async () => {
|
||
const one = { body: "机制", cells: [{ factor: "p1", range: r(1, 2, 3) }] };
|
||
const reason = await rejected({ kind: "explanation", content: { explanations: [one, one] }, checklist: { kind: "explanation", cells: ["p1"], maxExplanations: 1 }, known: null, jevModel: MODEL });
|
||
assert.match(reason, /交了 2 条解释，最多 1 条/);
|
||
});
|
||
|
||
test("规则 7：志愿个数够、不重复、都在入口清单里", async () => {
|
||
const checklist: ChecklistOf<"claim"> = { kind: "claim", entries: ["e-a1-1", "e-a2-1"], minChoices: 2 };
|
||
assert.match(await rejected({ kind: "claim", content: { choices: ["e-a1-1"] }, checklist, known: null, jevModel: MODEL }), /choices 至少要排 2 个入口，你排了 1 个/);
|
||
const reason = await rejected({ kind: "claim", content: { choices: ["e-a1-1", "e-a1-1", "e-a3-1"] }, checklist, known: null, jevModel: MODEL });
|
||
assert.match(reason, /choices 只能写 e-a1-1、e-a2-1；你写的 e-a3-1 不在里面/);
|
||
assert.match(reason, /choices 里 e-a1-1 写了不止一次/);
|
||
});
|
||
|
||
test("规则 8：挑刺针对的因子编号在被挑提案的因子范围内", async () => {
|
||
const reason = await rejected({ kind: "critique", content: { target: "p3", body: "挑刺" }, checklist: { kind: "critique", proposal: "r1-a1-1", factors: ["p1", "p2"] }, known: null, jevModel: MODEL });
|
||
assert.match(reason, /target 只能写 whole、p1、p2；你写的 p3 不在里面/);
|
||
});
|
||
|
||
const ORIGINAL: OriginalProposal = { factors: PROPOSAL.factors, safeguards: PROPOSAL.safeguards, metrics: PROPOSAL.metrics, body: PROPOSAL.body };
|
||
const RESPONSE_LIST: ChecklistOf<"response"> = { kind: "response", candidate: "r1-c1", critiques: ["r1-a1-1.a2.1", "r1-a1-1.a3.1"], ...RULES };
|
||
const response = (content: Submission<"response">): Incoming => ({ kind: "response", content, checklist: RESPONSE_LIST, known: { original: ORIGINAL }, jevModel: MODEL });
|
||
|
||
test("规则 9a：回应的答复恰好覆盖全部挑刺；原因里列出该写的全部编号（参考项目提交 378fe68）", async () => {
|
||
const reason = await rejected(response({ replies: [{ critique: "r1-a1-1.a2.2", reply: "改了" }], changes: {}, clauses: ["c"], body: "无" }));
|
||
assert.equal(reason, "这份提交没有收下，要改的地方：\n- replies 的挑刺编号应当是 r1-a1-1.a2.1、r1-a1-1.a3.1；你写的 r1-a1-1.a2.2 不在里面；还缺 r1-a1-1.a2.1、r1-a1-1.a3.1");
|
||
});
|
||
|
||
const UNIFICATION: Submission<"unification"> = {
|
||
...ORIGINAL,
|
||
replies: [],
|
||
clauses: ["c1 条"],
|
||
sources: [{ proposal: "r1-a1-1", coverage: "全在" }],
|
||
};
|
||
const unification = (content: Submission<"unification">): Incoming => ({
|
||
kind: "unification",
|
||
content,
|
||
checklist: { kind: "unification", candidate: "r1-c1", critiques: [], sources: ["r1-a1-1", "r1-a2-1"], ...RULES },
|
||
known: null,
|
||
jevModel: MODEL,
|
||
});
|
||
|
||
test("规则 9b：统一的 sources 恰好覆盖被合并的原提案；没有挑刺时说交空数组", async () => {
|
||
const reason = await rejected(unification({ ...UNIFICATION, replies: [{ critique: "x.a1.1", reply: "答" }] }));
|
||
assert.match(reason, /replies 的挑刺编号应当是 没有（交空数组）；你写的 x.a1.1 不在里面/);
|
||
assert.match(reason, /sources 的原提案编号应当是 r1-a1-1、r1-a2-1；还缺 r1-a2-1/);
|
||
});
|
||
|
||
test("规则 2、3、4、10 对拼好的协议查：回应改出来的因子超过上限也退回", async () => {
|
||
const reason = await rejected(response({ replies: [{ critique: "r1-a1-1.a2.1", reply: "答" }, { critique: "r1-a1-1.a3.1", reply: "答" }], changes: { factors: [factor("a", null), factor("b", null), factor("c", null)] }, clauses: ["c"], body: "无" }));
|
||
assert.match(reason, /factors 有 3 个因子，超过上限 2 个/);
|
||
});
|
||
|
||
test("规则 2、3、4、10 对统一自己的字段查：因子超上限、保障叫 main、登记已占用的编号都退回", async () => {
|
||
const reason = await rejected(unification({ ...UNIFICATION, factors: [factor("a", null), factor("b", null), factor("c", null)], safeguards: ["main"], metrics: { defined: [metric("wall-seconds")], reused: [] }, sources: [{ proposal: "r1-a1-1", coverage: "全在" }, { proposal: "r1-a2-1", coverage: "全在" }] }));
|
||
assert.match(reason, /factors 有 3 个因子，超过上限 2 个/);
|
||
assert.match(reason, /main/);
|
||
assert.match(reason, /metrics.defined 的编号 wall-seconds 已被占用/);
|
||
});
|
||
|
||
test("规则 10：reused 要在可引用清单里；新登记的编号不重复，也不能和已占用的重名（含答案路径两个保留键）", async () => {
|
||
assert.match(await rejected(proposal({ ...PROPOSAL, metrics: { defined: [], reused: ["kick-count"] } }, 0)), /metrics.reused 只能写 r1-c1\.restart-count、r1-c2\.restart-count；你写的 kick-count 不在里面/);
|
||
const reason = await rejected(proposal({ ...PROPOSAL, metrics: { defined: [metric("a"), metric("a"), metric("wall-seconds"), metric("answer-path")], reused: [] } }, 0));
|
||
assert.match(reason, /metrics.defined 里编号 a 写了不止一次/);
|
||
assert.match(reason, /metrics.defined 的编号 wall-seconds 已被占用/);
|
||
assert.match(reason, /metrics.defined 的编号 answer-path 已被占用/);
|
||
});
|
||
|
||
test("规则 10：引用写短名而不是全名时退回，原因里列出能引用的全名；新登记的短名和已登记的全名同名不算占用", async () => {
|
||
const reason = await rejected(proposal({ ...PROPOSAL, metrics: { defined: [], reused: ["restart-count"] } }, 0));
|
||
assert.equal(reason, "这份提交没有收下，要改的地方：\n- metrics.reused 只能写 r1-c1.restart-count、r1-c2.restart-count；你写的 restart-count 不在里面");
|
||
const v = await receive(proposal({ ...PROPOSAL, metrics: { defined: [metric("restart-count")], reused: ["r1-c1.restart-count"] } }, 0));
|
||
assert.equal(v.accepted, true);
|
||
});
|
||
|
||
test("提案条数：已收下 M 条再交就退回", async () => {
|
||
assert.match(await rejected(proposal(PROPOSAL, 2)), /本阶段你已经收下 2 条提案，每人最多 2 条/);
|
||
});
|
||
|
||
const TABLE = {
|
||
factorLocations: [{ factor: "p1", locations: ["solve.py:10-20"] }, { factor: "p2", locations: ["solve.py:30"] }],
|
||
safeguardLocations: [{ safeguard: "guard", locations: ["solve.py"] }],
|
||
unassigned: [],
|
||
concerns: "无",
|
||
body: "无",
|
||
};
|
||
const FROZEN = { candidate: "r1-c1", clauses: ["c1", "c2"], factors: ["p1", "p2"], safeguards: ["guard"] };
|
||
const implementation = (code: Extract<Incoming, { kind: "implementation" }>["known"]["code"], content: Submission<"implementation">): Incoming => ({
|
||
kind: "implementation",
|
||
content,
|
||
checklist: { kind: "implementation", ...FROZEN },
|
||
known: { code },
|
||
jevModel: MODEL,
|
||
});
|
||
const SAVED = { saved: true as const, protectedFiles: [], hunks: [], protocol: { body: "协议", metrics: [] } };
|
||
|
||
test("规则 11：位置表覆盖每个因子和每条保障；起始行不大于结束行", async () => {
|
||
const reason = await rejected(
|
||
implementation(SAVED, { ...TABLE, factorLocations: [{ factor: "p1", locations: ["solve.py:20-10"] }], safeguardLocations: [], unassigned: [{ location: "util.py:5-3", note: "日志" }] }),
|
||
);
|
||
assert.match(reason, /factorLocations 的因子应当是 p1、p2；还缺 p2/);
|
||
assert.match(reason, /safeguardLocations 的保障应当是 guard；还缺 guard/);
|
||
assert.match(reason, /代码位置 solve.py:20-10 的起始行大于结束行/);
|
||
assert.match(reason, /代码位置 util.py:5-3 的起始行大于结束行/);
|
||
});
|
||
|
||
test("改了不许改的文件：退回，原因里写出是哪几个文件（实现、修复、消融实现都查）", async () => {
|
||
const reason = await rejected(implementation({ ...SAVED, protectedFiles: ["validate.py", "harness.py"] }, TABLE));
|
||
assert.equal(reason, "这份提交没有收下，要改的地方：\n- 改了不许改的文件：validate.py、harness.py。把它们改回本批基线的样子再交");
|
||
const repair = await rejected({ kind: "repair", content: TABLE, checklist: { kind: "repair", ...FROZEN }, known: { code: { ...SAVED, protectedFiles: ["validate.py"] } }, jevModel: MODEL });
|
||
assert.match(repair, /改了不许改的文件：validate.py/);
|
||
const ablation = await rejected({
|
||
kind: "ablationImplementation",
|
||
content: { alsoOff: [], cannotRun: null, concerns: "无" },
|
||
checklist: { kind: "ablationImplementation", cell: "r1-c1-p1", off: "p1", factors: ["p1", "p2"] },
|
||
known: { code: { saved: true, protectedFiles: ["validate.py"] } },
|
||
jevModel: MODEL,
|
||
});
|
||
assert.match(ablation, /改了不许改的文件：validate.py/);
|
||
});
|
||
|
||
test("代码存不了：退回，原样写 workspace 给的路径和原因", async () => {
|
||
const reason = await rejected(implementation({ saved: false, path: "data/link", reason: "符号链接，代码版本只存普通文件和目录" }, TABLE));
|
||
assert.equal(reason, "这份提交没有收下，要改的地方：\n- 代码存不成版本：data/link（符号链接，代码版本只存普通文件和目录）。改掉它再交");
|
||
});
|
||
|
||
test("规则 12：核验的条款、因子、保障、没有归属的改动逐项全覆盖", async () => {
|
||
const reason = await rejected({
|
||
kind: "verification",
|
||
content: {
|
||
clauses: [{ clause: "c1", found: true, where: "a" }],
|
||
factors: [{ factor: "p1", found: true, where: "a" }, { factor: "p2", found: true, where: "a" }, { factor: "p2", found: true, where: "a" }],
|
||
safeguards: [],
|
||
unassigned: [{ location: "util.py:3", outOfScope: false, reason: "日志" }],
|
||
body: "无",
|
||
},
|
||
checklist: { kind: "verification", ...FROZEN, unassigned: ["util.py:1-4"] },
|
||
known: null,
|
||
jevModel: MODEL,
|
||
});
|
||
assert.match(reason, /clauses 的实现条款应当是 c1、c2；还缺 c2/);
|
||
assert.match(reason, /factors 的因子应当是 p1、p2；p2 写了不止一次/);
|
||
assert.match(reason, /safeguards 的保障应当是 guard；还缺 guard/);
|
||
assert.match(reason, /unassigned 的改动位置应当是 util.py:1-4；你写的 util.py:3 不在里面；还缺 util.py:1-4/);
|
||
});
|
||
|
||
test("规则 13：cannotRun 时 alsoOff 为空；alsoOff 不含本格的因子、都在因子清单里、不重复", async () => {
|
||
const checklist: ChecklistOf<"ablationImplementation"> = { kind: "ablationImplementation", cell: "r1-c1-p1", off: "p1", factors: ["p1", "p2"] };
|
||
const both = await rejected({ kind: "ablationImplementation", content: { alsoOff: ["p2"], cannotRun: "跑不起来", concerns: "无" }, checklist, known: { code: null }, jevModel: MODEL });
|
||
assert.match(both, /写了 cannotRun（这一格跑不起来）时 alsoOff 要交空数组/);
|
||
const reason = await rejected({ kind: "ablationImplementation", content: { alsoOff: ["p1", "p5", "p2", "p2"], cannotRun: null, concerns: "无" }, checklist, known: { code: { saved: true, protectedFiles: [] } }, jevModel: MODEL });
|
||
assert.match(reason, /alsoOff 不能写本格要关的因子 p1/);
|
||
assert.match(reason, /alsoOff 只能写 p2；你写的 p5 不在里面/);
|
||
assert.doesNotMatch(reason, /只能写 p1/);
|
||
assert.match(reason, /alsoOff 里 p2 写了不止一次/);
|
||
});
|
||
|
||
test("规则 14：消融核验覆盖每个因子", async () => {
|
||
const reason = await rejected({ kind: "ablationVerification", content: { factors: [{ factor: "p1", off: true, where: "a" }] }, checklist: { kind: "ablationVerification", cell: "r1-c1-p1", factors: ["p1", "p2"] }, known: null, jevModel: MODEL });
|
||
assert.match(reason, /factors 的因子应当是 p1、p2；还缺 p2/);
|
||
});
|
||
|
||
test("一份提交的几处问题一次列全，每条一行", async () => {
|
||
const reason = await rejected(proposal({ ...PROPOSAL, safeguards: ["main"], authorEstimate: r(3, 2, 1) }, 2));
|
||
const lines = reason.split("\n");
|
||
assert.equal(lines.length, 4, reason);
|
||
assert.equal(lines[0], "这份提交没有收下，要改的地方：");
|
||
assert.ok(lines.slice(1).every((l) => l.startsWith("- ")), reason);
|
||
assert.match(reason, /main/);
|
||
assert.match(reason, /严格递增/);
|
||
});
|
||
|
||
// ---------------------------------------------------------------------------
|
||
// 拼冻结协议
|
||
// ---------------------------------------------------------------------------
|
||
|
||
test("回应拼协议：changes 只改保障时，因子、指标、正文取原提案的，实现条款取回应自己的；答复不进协议", () => {
|
||
const p = protocolBody({ kind: "response", content: { replies: [{ critique: "x", reply: "答" }], changes: { safeguards: ["clip"] }, clauses: ["条款一"], body: "答复正文" }, original: ORIGINAL });
|
||
assert.deepEqual(p, {
|
||
factors: [factor("多起点", null), factor("扰动", "r1-c1-p2")],
|
||
safeguards: ["clip"],
|
||
metrics: { defined: [metric("kick-count")], reused: ["r1-c1.restart-count"] },
|
||
body: "提案正文",
|
||
clauses: ["条款一"],
|
||
});
|
||
});
|
||
|
||
test("回应拼协议：changes 改了正文时用新正文", () => {
|
||
const p = protocolBody({ kind: "response", content: { replies: [], changes: { body: "新正文" }, clauses: ["条款一"], body: "无" }, original: ORIGINAL });
|
||
assert.equal(p.body, "新正文");
|
||
assert.deepEqual(p.safeguards, ["guard"]);
|
||
});
|
||
|
||
test("统一拼协议：全用统一提交自己的字段，不叠原提案，不带 replies、sources", () => {
|
||
const content: Submission<"unification"> = { factors: [factor("合并后的因子", null)], safeguards: [], metrics: { defined: [], reused: [] }, body: "统一正文", replies: [{ critique: "x", reply: "答" }], clauses: ["统一条款"], sources: [{ proposal: "r1-a1-1", coverage: "全在" }] };
|
||
assert.deepEqual(protocolBody({ kind: "unification", content }), { factors: [factor("合并后的因子", null)], safeguards: [], metrics: { defined: [], reused: [] }, body: "统一正文", clauses: ["统一条款"] });
|
||
});
|
||
|
||
// ---------------------------------------------------------------------------
|
||
// 核验、消融核验通过没有
|
||
// ---------------------------------------------------------------------------
|
||
|
||
const VERIFICATION: Submission<"verification"> = {
|
||
clauses: [{ clause: "c1", found: true, where: "a" }],
|
||
factors: [{ factor: "p1", found: true, where: "a" }],
|
||
safeguards: [{ safeguard: "guard", found: true, where: "a" }],
|
||
unassigned: [{ location: "util.py:3", outOfScope: false, reason: "日志" }],
|
||
body: "无",
|
||
};
|
||
|
||
test("核验：全部找到、没有越界就通过", () => {
|
||
assert.deepEqual(verificationOutcome(VERIFICATION), { passed: true, clauses: [], factors: [], safeguards: [], outOfScope: [] });
|
||
});
|
||
|
||
test("核验：一项没落实或一条判为越界就不通过，分开列出", () => {
|
||
const v = { ...VERIFICATION, safeguards: [{ safeguard: "guard", found: false, where: "看了 solve.py" }], unassigned: [{ location: "util.py:3", outOfScope: true, reason: "改了计时" }] };
|
||
assert.deepEqual(verificationOutcome(v), { passed: false, clauses: [], factors: [], safeguards: ["guard"], outOfScope: ["util.py:3"] });
|
||
});
|
||
|
||
const ABLATION_V = (offs: Record<string, boolean>): Submission<"ablationVerification"> => ({
|
||
factors: Object.entries(offs).map(([factor, off]) => ({ factor, off, where: "a" })),
|
||
});
|
||
|
||
test("消融核验：要关的和一起关的都关着、其余照留，就通过", () => {
|
||
assert.deepEqual(ablationOutcome(ABLATION_V({ p1: true, p2: false, p3: true }), "p1", ["p3"]), { passed: true, leftOn: [], wronglyOff: [] });
|
||
});
|
||
|
||
test("消融核验：该关的照留、该留的关了，就不通过", () => {
|
||
assert.deepEqual(ablationOutcome(ABLATION_V({ p1: false, p2: true, p3: false }), "p1", []), { passed: false, leftOn: ["p1"], wronglyOff: ["p2"] });
|
||
});
|
||
|
||
// ---------------------------------------------------------------------------
|
||
// 差异块的退回文字（假 Jev）
|
||
// ---------------------------------------------------------------------------
|
||
|
||
function fakeJev(t: TestContext, yes: (question: string, diff: string) => boolean): void {
|
||
const key = process.env["TYPESAFE_API_KEY"];
|
||
process.env["TYPESAFE_API_KEY"] = "test-key";
|
||
t.after(() => {
|
||
if (key === undefined) delete process.env["TYPESAFE_API_KEY"];
|
||
else process.env["TYPESAFE_API_KEY"] = key;
|
||
});
|
||
let n = 0;
|
||
t.mock.method(globalThis, "fetch", async (_url: string, init: RequestInit) => {
|
||
const body = JSON.parse(init.body as string);
|
||
const answers = Object.fromEntries(Object.keys(body.questions).map((q) => [q, { noul: yes(q, body.state.change?.diff ?? "") ? 0.99 : 0.01 }]));
|
||
n += 1;
|
||
return new Response(JSON.stringify({ answers }), { status: 200, headers: { "x-typesafe-request-id": `req_${n}` } });
|
||
});
|
||
}
|
||
|
||
test("实现：每块问一次，判断全交回；退回只列判是的几问，写出文件、第几块、附差异原文，原文含 ``` 时围栏加长", async (t) => {
|
||
fakeJev(t, (q, diff) => q === "unseeded_randomness" && diff.includes("random"));
|
||
const hunks = [
|
||
{ path: "solve.py", change: "modified" as const, text: "@@ -1 +1 @@\n-a\n+b" },
|
||
{ path: "solve.py", change: "modified" as const, text: "@@ -9 +9 @@\n-x = 1\n+x = random() # ```" },
|
||
];
|
||
const v = await receive(implementation({ ...SAVED, hunks }, TABLE));
|
||
assert.equal(v.accepted, false);
|
||
assert.equal(v.jev!.length, 10, "两块各五问");
|
||
assert.equal(
|
||
(v as Extract<Verdict, { accepted: false }>).reason,
|
||
[
|
||
"这份提交没有收下，Jev 判出下面几处有问题：",
|
||
"",
|
||
"- solve.py（改动的文件）的第 2 块：unseeded_randomness 概率 0.99（阈值 " + v.jev!.find((j) => j.question === "unseeded_randomness")!.threshold + "）",
|
||
"",
|
||
"````diff",
|
||
"@@ -9 +9 @@\n-x = 1\n+x = random() # ```",
|
||
"````",
|
||
].join("\n"),
|
||
);
|
||
});
|
||
|
||
test("回应全判否：收下，交回拼好的协议和三问的判断", async (t) => {
|
||
fakeJev(t, () => false);
|
||
const v = await receive(response({ replies: [{ critique: "r1-a1-1.a2.1", reply: "答" }, { critique: "r1-a1-1.a3.1", reply: "答" }], changes: { body: "新正文" }, clauses: ["条款一"], body: "无" }));
|
||
assert.equal(v.accepted, true);
|
||
assert.equal(v.jev!.length, 3);
|
||
assert.deepEqual(v.accepted && v.protocol, {
|
||
factors: [factor("多起点", null), factor("扰动", "r1-c1-p2")],
|
||
safeguards: ["guard"],
|
||
metrics: { defined: [metric("kick-count")], reused: ["r1-c1.restart-count"] },
|
||
body: "新正文",
|
||
clauses: ["条款一"],
|
||
});
|
||
});
|
||
|
||
test("统一的冻结协议有一问判是：退回，原因只列这一问", async (t) => {
|
||
fakeJev(t, (q) => q === "changes_measurement");
|
||
const v = await receive(unification({ ...UNIFICATION, sources: [{ proposal: "r1-a1-1", coverage: "全在" }, { proposal: "r1-a2-1", coverage: "全在" }] }));
|
||
assert.equal(v.accepted, false);
|
||
assert.equal(v.jev!.length, 3);
|
||
const threshold = v.jev!.find((j) => j.question === "changes_measurement")!.threshold;
|
||
assert.equal((v as Extract<Verdict, { accepted: false }>).reason, `这份提交没有收下，Jev 判出下面几处有问题：\n\n- 冻结协议：changes_measurement 概率 0.99（阈值 ${threshold}）`);
|
||
});
|
||
|
||
test("实现有差异块、全判否：收下，jev 是全部判断", async (t) => {
|
||
fakeJev(t, () => false);
|
||
const v = await receive(implementation({ ...SAVED, hunks: [{ path: "solve.py", change: "modified", text: "@@ -1 +1 @@\n-a\n+b" }] }, TABLE));
|
||
assert.equal(v.accepted, true);
|
||
assert.equal(v.jev!.length, 5);
|
||
});
|
||
|
||
test("消融实现的代码存不成版本：退回", async () => {
|
||
const reason = await rejected({
|
||
kind: "ablationImplementation",
|
||
content: { alsoOff: [], cannotRun: null, concerns: "无" },
|
||
checklist: { kind: "ablationImplementation", cell: "r1-c1-p1", off: "p1", factors: ["p1", "p2"] },
|
||
known: { code: { saved: false, path: "bin/x", reason: "符号链接，代码版本只存普通文件和目录" } },
|
||
jevModel: MODEL,
|
||
});
|
||
assert.match(reason, /代码存不成版本：bin\/x（符号链接/);
|
||
});
|
||
|
||
// ---------------------------------------------------------------------------
|
||
// 诊断（方法.md 4.3 诊 1–诊 4）
|
||
// ---------------------------------------------------------------------------
|
||
|
||
const DX_EXPERIMENT = (name: string, missing: "recordMissing" | "invalidateRun" = "recordMissing", ids: string[] = []) => ({
|
||
name,
|
||
spot: "solve.py 的起点生成",
|
||
body: "换成更强的起点，交出答案",
|
||
metrics: ids.map((id) => ({ ...metric(id), missing })),
|
||
});
|
||
const DX_PROPOSAL = (over: Partial<Submission<"diagnosisProposal">> = {}): Submission<"diagnosisProposal"> => ({
|
||
explanations: [{ body: "卡在起点" }, { body: "卡在收尾" }],
|
||
experiments: [DX_EXPERIMENT("强起点"), DX_EXPERIMENT("拿掉收尾", "recordMissing", ["raw-score"])],
|
||
...over,
|
||
});
|
||
const DX_LIST: ChecklistOf<"diagnosisProposal"> = { kind: "diagnosisProposal", maxExplanations: 2, maxExperiments: 2, takenMetricIds: ["sum-of-radii", "answer-path"] };
|
||
const dxProposal = (content: Submission<"diagnosisProposal">): Incoming => ({ kind: "diagnosisProposal", content, checklist: DX_LIST, known: null, jevModel: MODEL });
|
||
|
||
test("诊断解释与实验：解释 1 至 M 条、实验 1 至 A 个；超过就退回", async () => {
|
||
const many = DX_PROPOSAL({ explanations: [{ body: "a" }, { body: "b" }, { body: "c" }], experiments: [DX_EXPERIMENT("1"), DX_EXPERIMENT("2"), DX_EXPERIMENT("3")] });
|
||
const reason = await rejected(dxProposal(many));
|
||
assert.match(reason, /交了 3 条诊断解释，最多 2 条/);
|
||
assert.match(reason, /交了 3 个诊断实验，最多 2 个/);
|
||
});
|
||
|
||
test("诊断解释与实验：登记的指标只能是 recordMissing；同一个实验里不重名、不占用已有编号", async () => {
|
||
const reason = await rejected(
|
||
dxProposal(
|
||
DX_PROPOSAL({
|
||
experiments: [
|
||
DX_EXPERIMENT("甲", "invalidateRun", ["raw-score"]),
|
||
{ ...DX_EXPERIMENT("乙"), metrics: [metric("a"), metric("a"), metric("answer-path")] },
|
||
],
|
||
}),
|
||
),
|
||
);
|
||
assert.match(reason, /第 1 个实验的指标 raw-score 选了 invalidateRun：诊断实验里的指标只能选 recordMissing/);
|
||
assert.match(reason, /第 2 个实验的 metrics 里编号 a 写了不止一次/);
|
||
assert.match(reason, /第 2 个实验的指标编号 answer-path 已被占用/);
|
||
});
|
||
|
||
test("诊断解释与实验：对每个实验问冻结三问；两个实验用同一个指标短名没问题（冻结时各带自己实验的前缀）", async (t) => {
|
||
fakeJev(t, () => false);
|
||
const v = await receive(dxProposal(DX_PROPOSAL({ experiments: [DX_EXPERIMENT("甲", "recordMissing", ["x"]), DX_EXPERIMENT("乙", "recordMissing", ["x"])] })));
|
||
assert.equal(v.accepted, true);
|
||
assert.equal(v.jev!.length, 6, "两个实验各三问");
|
||
assert.deepEqual(v.jev!.map((j) => j.question), ["self_contradictory", "changes_measurement", "too_vague", "self_contradictory", "changes_measurement", "too_vague"]);
|
||
});
|
||
|
||
test("诊断解释与实验：Jev 判是就退回，原因写明第几个实验和哪一问", async (t) => {
|
||
fakeJev(t, (q) => q === "too_vague");
|
||
const v = await receive(dxProposal(DX_PROPOSAL()));
|
||
assert.equal(v.accepted, false);
|
||
const reason = (v as Extract<Verdict, { accepted: false }>).reason;
|
||
assert.match(reason, /Jev 判出下面几处有问题/);
|
||
assert.match(reason, /- 第 1 个诊断实验（强起点）：too_vague 概率 0\.99/);
|
||
assert.match(reason, /- 第 2 个诊断实验（拿掉收尾）：too_vague 概率 0\.99/);
|
||
});
|
||
|
||
const DX_PRED_LIST: ChecklistOf<"diagnosisPrediction"> = { kind: "diagnosisPrediction", explanations: ["r1-dx1", "r1-dx2"], experiments: ["d1", "d2"] };
|
||
const dxPred = (predictions: Array<[string, string, ReturnType<typeof r>]>): Incoming => ({
|
||
kind: "diagnosisPrediction",
|
||
content: { predictions: predictions.map(([explanation, experiment, range]) => ({ explanation, experiment, range })), body: "理由" },
|
||
checklist: DX_PRED_LIST,
|
||
known: null,
|
||
jevModel: MODEL,
|
||
});
|
||
|
||
test("诊断预测：自己每条解释 × 每个合并后的实验，不多不少；收下时不问 Jev", async () => {
|
||
const all: Array<[string, string, ReturnType<typeof r>]> = [
|
||
["r1-dx1", "d1", r(1, 1.1, 1.2)], ["r1-dx1", "d2", r(1, 1.1, 1.2)], ["r1-dx2", "d1", r(1, 1.1, 1.2)], ["r1-dx2", "d2", r(1, 1.1, 1.2)],
|
||
];
|
||
assert.deepEqual(await receive(dxPred(all)), { accepted: true, jev: null, protocol: null });
|
||
const reason = await rejected(dxPred([all[0]!, all[0]!, all[1]!, ["r1-dx3", "d1", r(1, 1.1, 1.2)]]));
|
||
assert.match(reason, /predictions 的（解释:实验）应当是 r1-dx1:d1、r1-dx1:d2、r1-dx2:d1、r1-dx2:d2/);
|
||
assert.match(reason, /你写的 r1-dx3:d1 不在里面/);
|
||
assert.match(reason, /还缺 r1-dx2:d1、r1-dx2:d2/);
|
||
assert.match(reason, /r1-dx1:d1 写了不止一次/);
|
||
});
|
||
|
||
test("诊断预测：三个数要严格递增", async () => {
|
||
const reason = await rejected(dxPred([["r1-dx1", "d1", r(1, 1, 1.2)], ["r1-dx1", "d2", r(1, 1.1, 1.2)], ["r1-dx2", "d1", r(1, 1.1, 1.2)], ["r1-dx2", "d2", r(1.3, 1.2, 1.1)]]));
|
||
assert.match(reason, /r1-dx1:d1 的 range 的三个数要严格递增 q1 < q5 < q9，你写的是 1 \/ 1 \/ 1\.2/);
|
||
assert.match(reason, /r1-dx2:d2 的 range 的三个数要严格递增/);
|
||
});
|
||
|
||
const DX_CODE_BODY = { locations: [{ experiment: "d1", locations: ["solve.py:3-9"] }], concerns: "无", body: "无" };
|
||
type DxCode = Extract<Incoming, { kind: "diagnosisImplementation" }>["known"]["code"];
|
||
const dxCode = (kind: "diagnosisImplementation" | "diagnosisRepair", code: DxCode): Incoming =>
|
||
kind === "diagnosisImplementation"
|
||
? { kind, content: DX_CODE_BODY, checklist: { kind, experiments: ["d1"] }, known: { code }, jevModel: MODEL }
|
||
: { kind, content: DX_CODE_BODY, checklist: { kind, experiments: ["d1"] }, known: { code }, jevModel: MODEL };
|
||
|
||
test("诊断实现、修复：改了不许改的文件、代码存不了就退回；有差异块时每块问一次五问", async (t) => {
|
||
for (const kind of ["diagnosisImplementation", "diagnosisRepair"] as const) {
|
||
assert.match(await rejected(dxCode(kind, { ...SAVED, protectedFiles: ["validate.py"] })), /改了不许改的文件：validate\.py/);
|
||
assert.match(await rejected(dxCode(kind, { saved: false, path: "bin/x", reason: "符号链接" })), /代码存不成版本：bin\/x（符号链接）/);
|
||
}
|
||
fakeJev(t, () => false);
|
||
const hunks = [
|
||
{ path: "solve.py", change: "modified" as const, text: "@@ -1 +1 @@\n-a\n+b" },
|
||
{ path: "solve.py", change: "modified" as const, text: "@@ -9 +9 @@\n-c\n+d" },
|
||
];
|
||
const v = await receive(dxCode("diagnosisRepair", { ...SAVED, hunks }));
|
||
assert.equal(v.accepted, true);
|
||
assert.equal(v.jev!.length, 10, "两块各五问");
|
||
});
|
||
|
||
test("诊断核验：逐个实验的结果覆盖全部选中的实验", async () => {
|
||
const verification = (ids: string[]): Incoming => ({
|
||
kind: "diagnosisVerification",
|
||
content: { experiments: ids.map((experiment) => ({ experiment, followed: true, nothingElse: true, where: "solve.py:3" })), body: "无" },
|
||
checklist: { kind: "diagnosisVerification", experiments: ["d1", "d2"] },
|
||
known: null,
|
||
jevModel: MODEL,
|
||
});
|
||
assert.deepEqual(await receive(verification(["d2", "d1"])), { accepted: true, jev: null, protocol: null });
|
||
const reason = await rejected(verification(["d1", "d3"]));
|
||
assert.match(reason, /experiments 的实验应当是 d1、d2；你写的 d3 不在里面；还缺 d2/);
|
||
});
|
||
|
||
test("诊断核验通过没有：逐个实验判，一个不通过不影响别的实验", () => {
|
||
const v: Submission<"diagnosisVerification"> = {
|
||
experiments: [
|
||
{ experiment: "d1", followed: true, nothingElse: true, where: "a" },
|
||
{ experiment: "d2", followed: false, nothingElse: true, where: "没找到" },
|
||
{ experiment: "d3", followed: true, nothingElse: false, where: "顺手改了别处" },
|
||
{ experiment: "d4", followed: false, nothingElse: false, where: "都不对" },
|
||
],
|
||
body: "无",
|
||
};
|
||
assert.deepEqual(diagnosisOutcome(v), { passed: false, failing: ["d2", "d3", "d4"], notFollowed: ["d2", "d4"], touchedElse: ["d3", "d4"] });
|
||
assert.deepEqual(diagnosisOutcome({ experiments: [v.experiments[0]!], body: "无" }), { passed: true, failing: [], notFollowed: [], touchedElse: [] });
|
||
});
|