eval: 准备正向采纳的真实运行恢复与质量评估

新增单份串行启动入口、首个正向运行杀进程恢复证据、逐候选审阅导出与账目统计；参考只读副本分数对照。456项免费测试通过，尚未启动三份真实验收。同步结构说明与当前工作状态。
2026-10-04 04:25:00 -04:00
parent bf5b3f2358
commit b7be524cdc
10 changed files with 1329 additions and 14 deletions
@@ -0,0 +1,171 @@
+import { writeFileSync } from "node:fs";
+import { checkConfig } from "../../src/config/index.ts";
+import type { LedgerRecord } from "../../src/ledger/index.ts";
+import {
+  Board,
+  FORWARD_NOT_RUN,
+  type StudyRecord,
+} from "../../src/materials/index.ts";
+import { placement, runBalance } from "../../src/rules/index.ts";
+import { verificationOutcome } from "../../src/submissions/index.ts";
+import type { Range } from "../../src/vocab/index.ts";
+import { finalSubmissions, ofKind } from "./ledger.ts";
+
+export const FORWARD_STOP = "消融还没写（任务书 17）";
+export const jsonFile = (path: string, value: unknown): void =>
+  writeFileSync(path, `${JSON.stringify(value, null, 2)}\n`);
+export const frozenConfig = (records: LedgerRecord[]) =>
+  checkConfig((ofKind(records, "config")[0]!.content as StudyRecord).config);
+
+/** 区间边界只是报告口径，不参与研究决定；失败或测不到分数保持 null。 */
+export function scoreBand(score: number | null, range: Range): string | null {
+  if (score === null) return null;
+  return {
+    belowQ1: "<q1",
+    q1ToQ5: "[q1,q5)",
+    q5ToQ9: "[q5,q9]",
+    aboveQ9: ">q9",
+  }[placement(range, score)];
+}
+
+/** 核验要对应当时那版代码，不能拿最终修好后的代码替代历史被审代码。 */
+export function candidateHistory(records: LedgerRecord[], candidate: string) {
+  const code = ofKind(records, "code").filter(
+    (r) => r.content.object === candidate,
+  );
+  const submissions = ofKind(records, "submission").filter(
+    (r) => r.content.target === candidate,
+  );
+  const attempts = submissions.flatMap((r) => {
+    if (r.content.kind !== "implementation" && r.content.kind !== "repair")
+      return [];
+    // flow 在同一事务里先写 submission，紧接着写 code。
+    return [{ submission: r, code: code.find((c) => c.seq === r.seq + 1)! }];
+  });
+  const verifications = submissions.flatMap((r) =>
+    r.content.kind === "verification"
+      ? [
+          {
+            submission: r,
+            code: code.filter((c) => c.seq < r.seq).at(-1)!,
+            outcome: verificationOutcome(r.content.content),
+          },
+        ]
+      : [],
+  );
+  return { attempts, verifications };
+}
+
+/** 程序验收只读记录重算；基线也算正式运行，没启动的结果不扣次。 */
+export function forwardAccounts(records: LedgerRecord[]) {
+  const board = new Board(records);
+  const config = frozenConfig(records);
+  const opens = ofKind(records, "stageOpen");
+  const closes = ofKind(records, "stageClose");
+  const stages = opens.map((r) => {
+    const closing = closes.filter(
+      (c) => c.content.stageId === r.content.stageId,
+    );
+    return {
+      ...r.content,
+      openSeq: r.seq,
+      closeSeqs: closing.map((c) => c.seq),
+      pass: closing.length === 1 && closing[0]!.seq > r.seq,
+    };
+  });
+  const orphanCloses = closes.filter(
+    (c) => !opens.some((o) => o.content.stageId === c.content.stageId),
+  );
+  const started = ofKind(records, "runStarted");
+  const balance = runBalance(config.runs, board.runStates());
+  const runs = ofKind(records, "runPlanned").map((p) => ({
+    plan: p,
+    starts: started.filter((s) => s.content.run === p.content.run),
+    results: ofKind(records, "runResult").filter(
+      (r) => r.content.run === p.content.run,
+    ),
+  }));
+  const batch = board.decision("batch", 1);
+  const finals = finalSubmissions(records);
+  const candidates = (batch?.output.chosen ?? []).map((candidate) => {
+    const history = candidateHistory(records, candidate);
+    const events = ofKind(records, "event").filter(
+      (r) =>
+        r.content.event === FORWARD_NOT_RUN &&
+        r.content.objects.includes(candidate),
+    );
+    const candidateRuns = runs.filter(
+      (r) =>
+        r.plan.content.purpose === "forward" &&
+        r.plan.content.round === 1 &&
+        r.plan.content.object === candidate,
+    );
+    const verificationCount = finals.filter(
+      (r) =>
+        r.content.target === candidate && r.content.kind === "verification",
+    ).length;
+    const repairCount = finals.filter(
+      (r) => r.content.target === candidate && r.content.kind === "repair",
+    ).length;
+    const lastPassed = history.verifications.at(-1)?.outcome.passed ?? null;
+    const exhausted = lastPassed === false && repairCount >= config.maxRepairs;
+    return {
+      candidate,
+      verificationCount,
+      repairCount,
+      lastPassed,
+      exhausted,
+      notRunEvents: events,
+      started: candidateRuns.some((r) => r.starts.length > 0),
+      runs: candidateRuns,
+      pass: exhausted
+        ? events.length === 1 && candidateRuns.length === 0
+        : lastPassed === true &&
+          candidateRuns.some(
+            (r) => r.starts.length === 1 && r.results.length === 1,
+          ),
+    };
+  });
+  const adoption =
+    ofKind(records, "decision").find(
+      (r) => r.content.decision === "adopt" && r.content.round === 1,
+    ) ?? null;
+  const halt =
+    ofKind(records, "event")
+      .filter((r) => r.content.event === "halted")
+      .at(-1) ?? null;
+  const checks = {
+    stagesPaired: stages.every((s) => s.pass) && orphanCloses.length === 0,
+    chargedOnce:
+      balance.charged === started.length &&
+      config.runs - balance.balance === started.length &&
+      balance.reserved === 0,
+    forwardSerial: runs
+      .filter((r) => r.plan.content.purpose === "forward")
+      .every(
+        (r, i, forwards) =>
+          i === 0 ||
+          (forwards[i - 1]!.results.length === 1 &&
+            forwards[i - 1]!.results[0]!.seq < r.plan.seq),
+      ),
+    runsFinished: runs.length > 0 && runs.every((r) => r.results.length === 1),
+    candidatesFinished:
+      candidates.length > 0 && candidates.every((c) => c.pass),
+    adoptedAndStopped:
+      adoption !== null &&
+      halt?.content.reason === FORWARD_STOP &&
+      halt.seq === records.at(-1)!.seq,
+  };
+  return {
+    initial: config.runs,
+    confirmedStarts: started.length,
+    balance,
+    runs,
+    candidates,
+    stages,
+    orphanCloses,
+    adoption,
+    halt,
+    checks,
+  };
+}
@@ -0,0 +1,77 @@
+import { readdirSync, readFileSync } from "node:fs";
+import { join } from "node:path";
+
+/** Pi 会话里的 research_check：请求和结果按 toolCallId 配对，不把超限请求算成执行。 */
+export function lightCheckUsage(sessionDir: string, maxCount: number) {
+  type Part = { type: string; id?: string; name?: string; text?: string };
+  type Message = {
+    role: string;
+    toolCallId?: string;
+    toolName?: string;
+    isError?: boolean;
+    content: Part[];
+  };
+  const requests: Array<{ id: string; file: string; line: number }> = [];
+  const results = new Map<
+    string,
+    { file: string; line: number; message: Message }
+  >();
+  const files = readdirSync(sessionDir)
+    .filter((f) => f.endsWith(".jsonl"))
+    .sort();
+  if (files.length === 0) throw new Error(`没有会话文件：${sessionDir}`);
+  for (const file of files) {
+    readFileSync(join(sessionDir, file), "utf8")
+      .split("\n")
+      .forEach((line, index) => {
+        if (!line.trim()) return;
+        const entry = JSON.parse(line) as { type: string; message?: Message };
+        const m = entry.message;
+        if (entry.type !== "message" || m === undefined) return;
+        if (m.role === "assistant") {
+          for (const p of m.content)
+            if (p.type === "toolCall" && p.name === "research_check")
+              requests.push({ id: p.id!, file, line: index + 1 });
+        } else if (m.role === "toolResult" && m.toolName === "research_check")
+          results.set(m.toolCallId!, { file, line: index + 1, message: m });
+      });
+  }
+  const checks = requests.map((request) => {
+    const result = results.get(request.id);
+    if (result === undefined)
+      return { request, result: null, status: "missingResult", used: null };
+    const text = result.message.content
+      .filter((p) => p.type === "text")
+      .map((p) => p.text)
+      .join("\n");
+    const source = { file: result.file, line: result.line, text };
+    if (result.message.isError)
+      return { request, result: source, status: "toolError", used: null };
+    if (text.startsWith("本次调用的轻量检查已用完"))
+      return { request, result: source, status: "deniedLimit", used: null };
+    const value = JSON.parse(text) as {
+      used: number;
+      maxCount: number;
+      seconds: number;
+      result: unknown;
+    };
+    return { request, result: source, status: "executed", used: value.used };
+  });
+  const used = checks.flatMap((c) => (c.used === null ? [] : [c.used]));
+  const unresolved = checks.some(
+    (c) => c.status === "missingResult" || c.status === "toolError",
+  );
+  return {
+    sessionDir,
+    maxCount,
+    requested: checks.length,
+    executed: used.length,
+    deniedLimit: checks.filter((c) => c.status === "deniedLimit").length,
+    // 工具报错或缺结果时不能猜实际执行量，也不当作 0 次通过。
+    pass:
+      !unresolved &&
+      used.length <= maxCount &&
+      used.every((n, i) => n === i + 1),
+    checks,
+  };
+}
@@ -0,0 +1,70 @@
+import { basename, dirname } from "node:path";
+import { DatabaseSync } from "node:sqlite";
+
+/** 只读已有副本；保留 incomplete 的数值及重复尝试，不冒充有效成绩。 */
+export function referenceScores(path: string) {
+  const study = basename(dirname(path));
+  const db = new DatabaseSync(path, { readOnly: true });
+  try {
+    type Fact<T> = { seq: number; content: T };
+    const facts = <T>(kind: string): Fact<T>[] =>
+      (
+        db
+          .prepare(
+            "SELECT created_seq, payload FROM facts WHERE kind = ? ORDER BY created_seq",
+          )
+          .all(kind) as Array<{ created_seq: number; payload: string }>
+      ).map((r) => ({
+        seq: r.created_seq,
+        content: JSON.parse(r.payload) as T,
+      }));
+    const links = facts<{
+      purpose: string;
+      roundId: string;
+      runId: string;
+      candidateId: string;
+    }>("run-link");
+    const observations = facts<{
+      runId: string;
+      status: string;
+      finalMetric: { value: number | null };
+      reasons: string[];
+    }>("run-observation");
+    const adoption = facts<{
+      roundId: string;
+      adoptedRunId: string | null;
+      newBaselineId: string;
+    }>("adoption").find((r) => r.content.roundId === `${study}.r1`)!;
+    return links
+      .filter(
+        (r) =>
+          r.content.purpose === "forward" &&
+          r.content.roundId === `${study}.r1`,
+      )
+      .map((link) => {
+        const observation = observations.find(
+          (r) => r.content.runId === link.content.runId,
+        )!;
+        return {
+          source: path,
+          study,
+          candidate: link.content.candidateId,
+          run: link.content.runId,
+          status: observation.content.status,
+          rawScore: observation.content.finalMetric.value,
+          validScore:
+            observation.content.status === "valid"
+              ? observation.content.finalMetric.value
+              : null,
+          adopted: adoption.content.adoptedRunId === link.content.runId,
+          newBaseline: adoption.content.newBaselineId,
+          linkSeq: link.seq,
+          observationSeq: observation.seq,
+          adoptionSeq: adoption.seq,
+          reasons: observation.content.reasons,
+        };
+      });
+  } finally {
+    db.close();
+  }
+}