InfoRSI/tests/jev.test.ts

import assert from "node:assert/strict";
import { mkdirSync, mkdtempSync, rmSync, writeFileSync } from "node:fs";
import { tmpdir } from "node:os";
import { join } from "node:path";
import { test, type TestContext } from "node:test";
import {
  checkExperiment,
  checkHunk,
  checkProtocol,
  codeHunks,
  judgeSameChange,
  judgeSameExperiment,
  type DiagnosisExperiment,
  type FrozenProtocol,
  type ProtocolRules,
} from "../src/jev/index.ts";
import { hash, type Json } from "../src/vocab/index.ts";
import { compareVersions, createRepository, saveVersion } from "../src/workspace/index.ts";

// jev 的测试（任务书 08 第七部分），不花钱：假 Jev 在测试自己的环境里造——把全局 fetch 换成假的，
// 产品代码照常向真实地址发请求，拿到的是这里造的回答。重试的等待把全局 setTimeout 换成立刻回调，并记下要等多久。

type Reply = { status: number; body: string; requestId: string | null };
type Sent = { url: string; auth: string | null; body: { model: string; state: any; questions: Record<string, any> } };

/** 装一个假 Jev：每次请求交给 answer 决定回什么，并记下发出去的请求。 */
function fakeJev(t: TestContext, answer: (body: Sent["body"], n: number) => Reply | Error): Sent[] {
  const sent: Sent[] = [];
  const key = process.env["TYPESAFE_API_KEY"];
  process.env["TYPESAFE_API_KEY"] = "test-key";
  t.after(() => {
    if (key === undefined) delete process.env["TYPESAFE_API_KEY"];
    else process.env["TYPESAFE_API_KEY"] = key;
  });
  t.mock.method(globalThis, "fetch", async (url: string, init: RequestInit) => {
    const body = JSON.parse(init.body as string);
    sent.push({ url, auth: new Headers(init.headers).get("authorization"), body });
    const reply = answer(body, sent.length);
    if (reply instanceof Error) throw reply;
    const headers = reply.requestId === null ? {} : { "x-typesafe-request-id": reply.requestId };
    return new Response(reply.body, { status: reply.status, headers });
  });
  return sent;
}

/** 正常回答：每一问的概率由 p 给，请求编号是 req_<第几次请求>。 */
function answerAll(p: (name: string) => number): (body: Sent["body"], n: number) => Reply {
  return (body, n) => ({
    status: 200,
    requestId: `req_${n}`,
    body: JSON.stringify({
      model: "jev-test",
      answers: Object.fromEntries(Object.keys(body.questions).map((name) => [name, { type: "noul", noul: p(name) }])),
    }),
  });
}

/** 把重试前的等待换成立刻继续，记下每次要等的毫秒数。 */
function instantWaits(t: TestContext): number[] {
  const waits: number[] = [];
  t.mock.method(globalThis, "setTimeout", (fn: () => void, ms: number) => {
    waits.push(ms);
    setImmediate(fn);
  });
  return waits;
}

const PROTOCOL: FrozenProtocol = {
  factors: [
    {
      name: "外圈环半径缩到 0.45",
      method: "把 construct_packing 里外圈环半径常数 0.7 改为 0.45。",
      keySettings: "环半径 0.45",
      offBehavior: "回到 0.7",
      continues: null,
    },
  ],
  safeguards: ["clip-radii"],
  metrics: {
    defined: [
      { id: "min-gap", meaning: "最小间隙", unit: "无", direction: "maximize", howMeasured: "最终解上所有圆对的最小间隙，测一次", missing: "recordMissing" },
    ],
    reused: ["c1.overlap"],
  },
  body: "### 不许改什么\n\nharness.py 不改。",
  clauses: ["外圈环半径常数改为 0.45（initial.py:36）"],
};

const RULES: ProtocolRules = {
  body: PROTOCOL.body,
  metrics: [{ id: "min-gap", howMeasured: "最终解上所有圆对的最小间隙，测一次", missing: "recordMissing" }],
};

const HUNK = { path: "initial.py", change: "modified", text: "@@ -1 +1 @@\n-a\n+b" } as const;

/** 参考项目 src/judge/questions.ts 的 JEV_THRESHOLDS 填圆那一套里这九问的值，抄自参考项目源码。 */
const REFERENCE_THRESHOLDS: Record<string, number> = {
  same_change: 0.3,
  self_contradictory: 0.5,
  changes_measurement: 0.21,
  too_vague: 0.78,
  unseeded_randomness: 0.55,
  reads_clock: 0.55,
  touches_measurement: 0.65,
  early_exit: 0.63,
  zero_for_unmeasured: 0.62,
};

test("概率超过阈值判是，等于或低于阈值判否；阈值是参考项目的值", async (t) => {
  // 冻结三问：第一问比阈值高 0.01，第二问正好等于阈值，第三问低 0.01。
  const offsets: Record<string, number> = { self_contradictory: 0.01, changes_measurement: 0, too_vague: -0.01 };
  const sent = fakeJev(t, answerAll((name) => Math.round(((REFERENCE_THRESHOLDS[name] as number) + (offsets[name] as number)) * 100) / 100));
  const result = await checkProtocol("jev-x", PROTOCOL);
  assert.deepEqual(
    result.judgments.map((j) => [j.question, j.threshold, j.yes, j.requestId]),
    [
      ["self_contradictory", 0.5, true, "req_1"],
      ["changes_measurement", 0.21, false, "req_2"],
      ["too_vague", 0.78, false, "req_3"],
    ],
  );
  assert.equal(result.flagged, true);
  // 冻结三问每问单独一次请求，材料相同；模型名照传；密钥只在请求头里。
  assert.deepEqual(
    sent.map((s) => Object.keys(s.body.questions)),
    [["self_contradictory"], ["changes_measurement"], ["too_vague"]],
  );
  for (const s of sent) {
    assert.deepEqual(s.body.state, sent[0]?.body.state);
    assert.equal(s.body.model, "jev-x");
    assert.equal(s.auth, "Bearer test-key");
    assert.equal(s.url, "https://api.typesafe.ai/v1/systemone");
    assert.ok(!JSON.stringify(s.body).includes("test-key"));
  }
});

test("一个差异块五问一次请求，判重一对一次请求；阈值都是参考项目的值", async (t) => {
  const sent = fakeJev(t, answerAll((name) => (name === "early_exit" ? 0.64 : 0.2)));
  const hunk = await checkHunk("jev-x", HUNK, RULES);
  assert.equal(sent.length, 1);
  assert.equal(hunk.flagged, true);
  assert.deepEqual(
    hunk.judgments.map((j) => [j.question, j.threshold, j.yes]),
    [
      ["unseeded_randomness", 0.55, false],
      ["reads_clock", 0.55, false],
      ["touches_measurement", 0.65, false],
      ["early_exit", 0.63, true],
      ["zero_for_unmeasured", 0.62, false],
    ],
  );
  const same = await judgeSameChange("jev-x", PROTOCOL.factors, PROTOCOL.factors);
  assert.equal(sent.length, 2);
  assert.deepEqual(same, { question: "same_change", questionHash: same.questionHash, threshold: 0.3, probability: 0.2, yes: false, requestId: "req_2" });
});

test("回答读不懂就报错，带上请求编号", async (t) => {
  const cases: Array<[string, Reply, RegExp]> = [
    ["没有概率", { status: 200, requestId: "req_a", body: JSON.stringify({ answers: {} }) }, /same_change 没有 0 到 1 之间的概率（request-id: req_a）/],
    [
      "概率大于 1",
      { status: 200, requestId: "req_b", body: JSON.stringify({ answers: { same_change: { noul: 1.2 } } }) },
      /same_change 没有 0 到 1 之间的概率（request-id: req_b）/,
    ],
    [
      "概率不是数",
      { status: 200, requestId: "req_c", body: JSON.stringify({ answers: { same_change: { noul: "0.4" } } }) },
      /same_change 没有 0 到 1 之间的概率（request-id: req_c）/,
    ],
    ["不是 JSON", { status: 200, requestId: "req_d", body: "<html>" }, /不是 JSON（request-id: req_d）/],
    ["没带请求编号", { status: 200, requestId: null, body: JSON.stringify({ answers: { same_change: { noul: 0.1 } } }) }, /没带请求编号/],
  ];
  for (const [what, reply, message] of cases) {
    await t.test(what, async (t) => {
      fakeJev(t, () => reply);
      await assert.rejects(judgeSameChange("jev-x", PROTOCOL.factors, PROTOCOL.factors), message);
    });
  }
});

test("调不通时按 2、10、30 秒重试，发满 4 次再报错", async (t) => {
  const waits = instantWaits(t);
  const sent = fakeJev(t, () => new TypeError("fetch failed"));
  await assert.rejects(checkHunk("jev-x", HUNK, RULES), /发了 4 次都失败/);
  assert.equal(sent.length, 4);
  assert.deepEqual(waits, [2_000, 10_000, 30_000]);
});

test("服务暂时出错后恢复就照常判；请求本身被拒不重试；没有密钥直接报错", async (t) => {
  await t.test("503 后恢复", async (t) => {
    const waits = instantWaits(t);
    const ok = answerAll(() => 0.9);
    const sent = fakeJev(t, (body, n) => (n === 1 ? { status: 503, requestId: "req_x", body: "busy" } : ok(body, n)));
    const j = await judgeSameChange("jev-x", PROTOCOL.factors, PROTOCOL.factors);
    assert.deepEqual([j.yes, j.requestId], [true, "req_2"]);
    assert.equal(sent.length, 2);
    assert.deepEqual(waits, [2_000]);
  });
  await t.test("422 直接报错", async (t) => {
    const sent = fakeJev(t, () => ({ status: 422, requestId: "req_y", body: "bad" }));
    await assert.rejects(judgeSameChange("jev-x", PROTOCOL.factors, PROTOCOL.factors), /拒绝了请求，不重试：HTTP 422（request-id: req_y）/);
    assert.equal(sent.length, 1);
  });
  await t.test("没有密钥", async (t) => {
    const sent = fakeJev(t, answerAll(() => 0.1));
    delete process.env["TYPESAFE_API_KEY"];
    await assert.rejects(judgeSameChange("jev-x", PROTOCOL.factors, PROTOCOL.factors), /TYPESAFE_API_KEY/);
    assert.equal(sent.length, 0);
  });
});

test("问题哈希把材料说明、措辞、示例、阈值一起算，改一个字就变", async (t) => {
  const sent = fakeJev(t, answerAll(() => 0.1));
  const judgments = [
    await judgeSameChange("jev-x", PROTOCOL.factors, PROTOCOL.factors),
    ...(await checkProtocol("jev-x", PROTOCOL)).judgments,
    ...(await checkHunk("jev-x", HUNK, RULES)).judgments,
  ];
  assert.equal(judgments.length, 9);
  assert.equal(new Set(judgments.map((j) => j.questionHash)).size, 9);
  // 从实际发出去的请求里取回每一问的材料说明和问题原文。
  const asked = new Map(sent.flatMap((s) => Object.entries(s.body.questions).map(([name, noul]) => [name, { task: s.body.state.task, noul }])));
  for (const j of judgments) {
    const { task, noul } = asked.get(j.question) as { task: string; noul: Json };
    const content = { task, name: j.question, noul, threshold: j.threshold };
    assert.equal(j.questionHash, hash(content));
    const changed = JSON.parse(JSON.stringify(content));
    changed.noul.criteria.true.examples[0] = `${changed.noul.criteria.true.examples[0]}x`;
    assert.notEqual(hash(changed), j.questionHash);
    assert.notEqual(hash({ ...content, threshold: j.threshold + 0.01 }), j.questionHash);
    assert.notEqual(hash({ ...content, task: `${task} ` }), j.questionHash);
  }
});

test("材料：给定带类型的输入，发出去的材料与手写预期一致", async (t) => {
  const sent = fakeJev(t, answerAll(() => 0.1));
  const other = [{ name: "六行六边形网格", method: "用 6 行六边形网格摆圆心。", keySettings: "6 行", continues: "f2" }];
  await judgeSameChange("jev-x", PROTOCOL.factors, other);
  await checkProtocol("jev-x", PROTOCOL);
  await checkHunk("jev-x", { path: "a.py", change: "added", text: "@@ -0,0 +1 @@\n+x" }, RULES);
  const states = sent.map((s) => s.body.state);
  const [dedup, freeze, , , diff] = states;
  for (const state of states) assert.equal(typeof state.task, "string");
  // 判重放名称、做法和关键设定：关掉时换成什么、延续哪个因子不放。材料说明里也讲了关键设定是什么（任务书 11 第三部分）。
  assert.match(dedup.task, /`keySettings`: the setting values that decide its effect/);
  assert.deepEqual(dedup, {
    task: dedup.task,
    proposal_a: { factors: [{ name: "外圈环半径缩到 0.45", method: "把 construct_packing 里外圈环半径常数 0.7 改为 0.45。", keySettings: "环半径 0.45" }] },
    proposal_b: { factors: [{ name: "六行六边形网格", method: "用 6 行六边形网格摆圆心。", keySettings: "6 行" }] },
  });
  assert.deepEqual(freeze, {
    task: freeze.task,
    protocol: {
      factors: [
        {
          name: "外圈环半径缩到 0.45",
          method: "把 construct_packing 里外圈环半径常数 0.7 改为 0.45。",
          keySettings: "环半径 0.45",
          offBehavior: "回到 0.7",
          continues: null,
        },
      ],
      safeguards: ["clip-radii"],
      body: "### 不许改什么\n\nharness.py 不改。",
      metrics: {
        defined: [
          { id: "min-gap", meaning: "最小间隙", unit: "无", direction: "maximize", howMeasured: "最终解上所有圆对的最小间隙，测一次", missing: "recordMissing" },
        ],
        reused: ["c1.overlap"],
      },
      clauses: ["外圈环半径常数改为 0.45（initial.py:36）"],
    },
  });
  assert.deepEqual(diff, {
    task: diff.task,
    protocol: {
      body: "### 不许改什么\n\nharness.py 不改。",
      metrics: [{ id: "min-gap", howMeasured: "最终解上所有圆对的最小间隙，测一次", missing: "recordMissing" }],
    },
    change: { path: "a.py", change: "added", diff: "@@ -0,0 +1 @@\n+x" },
  });
  assert.equal(new Set(states.map((s) => s.task)).size, 3);
});

function tempDir(t: TestContext): string {
  const dir = mkdtempSync(join(tmpdir(), "inforsi-jev-"));
  t.after(() => rmSync(dir, { recursive: true, force: true }));
  return dir;
}

test("把 workspace 的比较结果切成差异块，每块从 @@ 行起", (t) => {
  const root = tempDir(t);
  const repo = join(root, "v.git");
  createRepository(repo);
  const dir = join(root, "code");
  mkdirSync(dir);
  const lines = Array.from({ length: 30 }, (_, i) => `line ${i + 1}`);
  writeFileSync(join(dir, "a.py"), `${lines.join("\n")}\n`);
  writeFileSync(join(dir, "bin.dat"), Buffer.from([0, 1, 2]));
  const v1 = saveVersion(repo, dir, []);
  lines[1] = "changed 2";
  lines[25] = "changed 26";
  writeFileSync(join(dir, "a.py"), `${lines.join("\n")}\n`);
  writeFileSync(join(dir, "bin.dat"), Buffer.from([0, 9, 2]));
  writeFileSync(join(dir, "new.py"), "x = 1\n");
  const v2 = saveVersion(repo, dir, []);
  assert.ok(v1.saved && v2.saved);
  const hunks = codeHunks(compareVersions(repo, v1.version, v2.version));
  assert.deepEqual(
    hunks.map((h) => [h.path, h.change, h.text.split("\n")[0]]),
    [
      ["a.py", "modified", "@@ -1,5 +1,5 @@"],
      ["a.py", "modified", "@@ -23,7 +23,7 @@ line 22"],
      ["new.py", "added", "@@ -0,0 +1 @@"],
    ],
  );
  assert.equal(hunks[0]?.text, "@@ -1,5 +1,5 @@\n line 1\n-line 2\n+changed 2\n line 3\n line 4\n line 5");
  assert.equal(hunks[2]?.text, "@@ -0,0 +1 @@\n+x = 1");
});

// ---------------------------------------------------------------------------
// 诊断实验的判重和冻结检查（方法.md 4.3 诊 1、诊 2）
// ---------------------------------------------------------------------------

const EXPERIMENT: DiagnosisExperiment = {
  name: "拿掉收尾",
  spot: "solve.py 的收尾修复",
  body: "拿掉收尾修复，直接交出原始答案",
  metrics: [{ id: "raw-score", meaning: "收尾之前的分数", unit: "无", direction: "maximize", howMeasured: "收尾之前的分数", missing: "recordMissing" }],
};

test("诊断实验判重：一对一次请求，只给名称、动哪一处、怎么动；阈值暂用判重的 0.3", async (t) => {
  const sent = fakeJev(t, answerAll(() => 0.31));
  const other = { name: "强起点", spot: "solve.py 的起点", body: "换成更强的起点" };
  const j = await judgeSameExperiment("jev-x", EXPERIMENT, other);
  assert.equal(sent.length, 1);
  assert.deepEqual(Object.keys(sent[0]!.body.questions), ["same_experiment"]);
  assert.deepEqual(sent[0]!.body.state.experiment_a, { name: "拿掉收尾", spot: "solve.py 的收尾修复", body: "拿掉收尾修复，直接交出原始答案" });
  assert.deepEqual(sent[0]!.body.state.experiment_b, other);
  assert.deepEqual([j.question, j.threshold, j.yes, j.requestId], ["same_experiment", 0.3, true, "req_1"]);
});

test("诊断实验判重：概率等于阈值判否", async (t) => {
  fakeJev(t, answerAll(() => 0.3));
  assert.equal((await judgeSameExperiment("jev-x", EXPERIMENT, EXPERIMENT)).yes, false);
});

test("诊断实验冻结检查：三问各一次请求，材料相同，问法是诊断的变体；有一问判是就标出", async (t) => {
  const sent = fakeJev(t, answerAll((name) => (name === "changes_measurement" ? 0.22 : 0.1)));
  const result = await checkExperiment("jev-x", EXPERIMENT);
  assert.deepEqual(
    result.judgments.map((j) => [j.question, j.threshold, j.yes]),
    [["self_contradictory", 0.5, false], ["changes_measurement", 0.21, true], ["too_vague", 0.78, false]],
  );
  assert.equal(result.flagged, true);
  assert.deepEqual(sent.map((s) => Object.keys(s.body.questions)), [["self_contradictory"], ["changes_measurement"], ["too_vague"]]);
  for (const s of sent) assert.deepEqual(s.body.state, sent[0]!.body.state);
  assert.deepEqual(sent[0]!.body.state.experiment, {
    name: "拿掉收尾",
    spot: "solve.py 的收尾修复",
    body: "拿掉收尾修复，直接交出原始答案",
    metrics: [{ id: "raw-score", howMeasured: "收尾之前的分数", missing: "recordMissing" }],
  });
  // 材料说明和问法都是诊断的变体，哈希和冻结协议同名三问的不同。
  const protocol = await checkProtocol("jev-x", PROTOCOL);
  result.judgments.forEach((j, i) => assert.notEqual(j.questionHash, protocol.judgments[i]!.questionHash, j.question));
});