InfoRSI/tests/agents-pi.ts

import assert from "node:assert/strict";
import { existsSync, mkdirSync, mkdtempSync, readdirSync, readFileSync, writeFileSync } from "node:fs";
import { homedir, tmpdir } from "node:os";
import { join } from "node:path";
import { runSession, type SessionOutcome, type SessionTask, type SubmitAttempt } from "../src/agents/index.ts";
import type { ModelUsage } from "../src/vocab/index.ts";
import { fullConfig } from "./config-helpers.ts";

// agents 的真实模型测试（任务书 07 第六部分），npm run test:pi：要花钱、要联网，不放进 npm test。
// 用本机真实的 ~/.pi/agent（里面有扩展、技能、插件包），模型取测试配置里的 glm-5.3。
// 每次会话的结局、每次回复的用量、墙钟时间都完整打印；会话目录留在临时目录里，路径也打印出来。

const models = fullConfig().models;
const root = mkdtempSync(join(tmpdir(), "inforsi-agents-pi-"));
console.log(`临时目录：${root}`);
const agentDir = join(homedir(), ".pi", "agent");
for (const sub of ["extensions", "skills"]) {
  const dir = join(agentDir, sub);
  console.log(`本机 ${dir}：${existsSync(dir) ? readdirSync(dir).join(", ") : "（没有）"}`);
}
const settings = JSON.parse(readFileSync(join(agentDir, "settings.json"), "utf8")) as { packages?: unknown[]; compaction?: unknown };
console.log(`本机 settings.json：packages ${JSON.stringify(settings.packages)}；compaction ${JSON.stringify(settings.compaction)}`);

function dirs(name: string): { workDir: string; sessionDir: string } {
  const workDir = join(root, name, "work");
  const sessionDir = join(root, name, "sessions");
  mkdirSync(workDir, { recursive: true });
  mkdirSync(sessionDir, { recursive: true });
  return { workDir, sessionDir };
}

function show(usage: ModelUsage): string {
  const v = (m: ModelUsage["inputTokens"]) => (m.status === "measured" ? String(m.value) : `测不到（${m.reason}）`);
  return `输入 ${v(usage.inputTokens)}，缓存读 ${v(usage.cacheReadTokens)}，缓存写 ${v(usage.cacheWriteTokens)}，输出 ${v(usage.outputTokens)}，价格 ${v(usage.cost)}`;
}

/** 会话文件里助手回复的条数（Pi 的会话文件每行一条 JSON）。 */
function assistantReplies(sessionFile: string): number {
  return readFileSync(sessionFile, "utf8")
    .split("\n")
    .filter((line) => line.trim() !== "")
    .map((line) => JSON.parse(line) as { type?: string; message?: { role?: string } })
    .filter((entry) => entry.type === "message" && entry.message?.role === "assistant").length;
}

function report(name: string, outcome: SessionOutcome, usages: ModelUsage[], attempts: string[]): void {
  console.log(`\n===== ${name} =====`);
  console.log(`结局：${JSON.stringify(outcome)}`);
  console.log(`墙钟时间：${outcome.wallSeconds.toFixed(1)} 秒`);
  attempts.forEach((a, i) => console.log(`提交回调第 ${i + 1} 次：${a}`));
  usages.forEach((u, i) => console.log(`第 ${i + 1} 次回复的用量：${show(u)}`));
  assert.ok(outcome.sessionFile !== null, "会话文件写出来了");
  const replies = assistantReplies(outcome.sessionFile);
  console.log(`会话文件里助手回复 ${replies} 条，用量回调 ${usages.length} 次`);
  assert.equal(usages.length, replies, "每次回复都交了一份用量");
  assert.ok(usages.every((u) => u.inputTokens.status === "measured" && u.outputTokens.status === "measured"), "模型服务每次都报了用量");
}

const stop = new AbortController().signal;

// ---------------------------------------------------------------------------
// 研究者：交一份收尾，第一次故意退回，改了再交，收下
// ---------------------------------------------------------------------------

{
  const { workDir, sessionDir } = dirs("researcher");
  const readable = join(root, "researcher", "task");
  mkdirSync(readable);
  writeFileSync(join(readable, "README.md"), "填圆任务：在单位正方形里放 26 个圆，半径之和越大越好。\n");
  const usages: ModelUsage[] = [];
  const attempts: string[] = [];
  const task: SessionTask = {
    role: "researcher",
    model: models.researchers[0]!,
    systemPrompt: "你是一个研究小组里的研究者。你只能用给你的工具读资料和提交；只有调用 research_submit 才算交上了。",
    firstMessage: [
      "这是一次接口演练。请交一份收尾（research_submit，payload 里只有 body 一个字段）。",
      `body 写两三句话，说说这一轮的情况：可以先用 read_source 读一下 ${readable}/README.md 了解任务。`,
      "程序如果退回，按退回原因改好再交。被收下以后就结束，不用再做别的。",
    ].join("\n"),
    ...{ workDir, sessionDir },
    kind: "wrapUp",
    readableDirs: [readable],
    query: () => null,
    writeNote: (body) => attempts.push(`（写笔记 ${body.length} 字）`),
    usage: (u) => usages.push(u),
    stop,
    submit: async (attempt: SubmitAttempt<"wrapUp">) => {
      attempts.push(JSON.stringify(attempt));
      if (attempt.shape === "invalid") return { accepted: false, reason: attempt.reason };
      if (attempts.filter((a) => a.includes('"shape":"valid"')).length === 1) {
        return { accepted: false, reason: "收尾的最后一行必须是「下一步：继续在 p1 上试」，原样写上后整份重交" };
      }
      return { accepted: true, id: "s2" };
    },
  };
  const outcome = await runSession(task);
  report("研究者：先退回再收下", outcome, usages, attempts);
  assert.equal(outcome.status, "submitted");
  assert.ok(attempts.filter((a) => a.includes('"shape":"valid"')).length >= 2, "至少交了两次：一次被退回，一次收下");
}

// ---------------------------------------------------------------------------
// 实验员：在工作目录里写一个文件、跑一条 shell 命令，再交
// ---------------------------------------------------------------------------

{
  const { workDir, sessionDir } = dirs("experimenter");
  const usages: ModelUsage[] = [];
  const attempts: string[] = [];
  let checks = 0;
  const task: SessionTask = {
    role: "experimenter",
    model: models.experimenter,
    systemPrompt: "你是一个研究小组里的实验员，在自己的工作目录里写代码、跑代码。只有调用 research_submit 才算交上了。",
    firstMessage: [
      "这是一次接口演练，按顺序做三件事：",
      "1. 在工作目录里写一个 hello.py，运行时打印 hello inforsi。",
      "2. 用 bash 运行 python3 hello.py，确认输出。",
      "3. 用 research_submit 提交：payload 里 alsoOff 交空数组，cannotRun 交 null，concerns 写一句你运行时看到的输出。",
    ].join("\n"),
    ...{ workDir, sessionDir },
    kind: "ablationImplementation",
    query: () => null,
    lightCheck: async () => `第 ${++checks} 次轻量检查：通过`,
    usage: (u) => usages.push(u),
    stop,
    submit: async (attempt: SubmitAttempt<"ablationImplementation">) => {
      attempts.push(JSON.stringify(attempt));
      if (attempt.shape === "invalid") return { accepted: false, reason: attempt.reason };
      return existsSync(join(workDir, "hello.py"))
        ? { accepted: true, id: "s1" }
        : { accepted: false, reason: "工作目录里没有 hello.py" };
    },
  };
  const outcome = await runSession(task);
  report("实验员：写文件、跑 shell、再交", outcome, usages, attempts);
  assert.equal(outcome.status, "submitted");
  console.log(`hello.py：${JSON.stringify(readFileSync(join(workDir, "hello.py"), "utf8"))}`);
  const log = readFileSync(outcome.sessionFile!, "utf8");
  assert.match(log, /"name":"bash"/, "会话里调过 bash");
  assert.match(log, /hello inforsi/, "bash 的输出进了会话");
}

console.log("\n全部通过：两次会话的子会话核对都通过（没有加载扩展、技能、提示模板和上下文文件），都交上了，每次回复都有用量。");