401 lines
18 KiB
TypeScript
401 lines
18 KiB
TypeScript
import { existsSync, mkdirSync, readFileSync, writeFileSync } from "node:fs";
|
||
import { join, resolve } from "node:path";
|
||
import { DatabaseSync } from "node:sqlite";
|
||
import type { TaskMaterial } from "../../src/materials/index.ts";
|
||
import { diagnosisAccounts } from "../lib/diagnosis.ts";
|
||
import { cpuRatio, CPU_RATIO_FLAG } from "../lib/full.ts";
|
||
import { frozenConfig, jsonFile } from "../lib/forward.ts";
|
||
import { ofKind, readLedger } from "../lib/ledger.ts";
|
||
import { lightCheckUsage } from "../lib/light-checks.ts";
|
||
import { readTimeline } from "../lib/timeline.ts";
|
||
|
||
// 任务书 18 第 3.2 节第 2、4、5、6、7 样里能由账本算出的数字。只算事实、不作质量判断
|
||
// （第 1、3 样和"提案是否沿诊断方向"由全新上下文的子 Agent 读 eval/diagnosis/export.ts 的材料）。
|
||
// 第 4 样"沿这个方向"的列来自子 Agent 的标注文件 labels.json（可选）：
|
||
// { "<研究编号>": { "<轮次>": { "proposals": { "<提案编号>": "along"|"not" }, "candidates": { "<候选编号>": "along"|"not" } } } }
|
||
// node eval/diagnosis/stats.ts <全新绝对统计输出目录> [--labels <labels.json>] <研究输出目录>...
|
||
const args = process.argv.slice(2);
|
||
const output = args.shift();
|
||
let labelsPath: string | null = null;
|
||
if (args[0] === "--labels") {
|
||
args.shift();
|
||
labelsPath = args.shift() ?? null;
|
||
}
|
||
const dirs = args;
|
||
if (!output || dirs.length === 0 || output !== resolve(output)) {
|
||
throw new Error(
|
||
"用法：node eval/diagnosis/stats.ts <全新绝对统计输出目录> [--labels <labels.json>] <研究输出目录>...",
|
||
);
|
||
}
|
||
mkdirSync(output);
|
||
|
||
type Label = "along" | "not";
|
||
type Labels = Record<
|
||
string,
|
||
Record<
|
||
string,
|
||
{ proposals: Record<string, Label>; candidates: Record<string, Label> }
|
||
>
|
||
>;
|
||
const labels: Labels =
|
||
labelsPath === null ? {} : JSON.parse(readFileSync(labelsPath, "utf8"));
|
||
|
||
const f = (v: number | null | undefined, d = 6) =>
|
||
v === null || v === undefined ? "—" : v.toFixed(d);
|
||
const e3 = (v: number | null | undefined) =>
|
||
v === null || v === undefined ? "—" : v.toExponential(3);
|
||
const range = (r: { q1: number; q5: number; q9: number } | null) =>
|
||
r === null ? "—" : `${f(r.q1)}/${f(r.q5)}/${f(r.q9)}`;
|
||
|
||
// 旧研究（任务书 17 的 B、D）和参考项目逐轮采纳分数。
|
||
const OLD = "/home/iomgaa/inforsi-eval/2026-10-04-task17";
|
||
const oldTrajectory = (x: "b" | "d") => {
|
||
const stats = JSON.parse(
|
||
readFileSync(join(OLD, `stats-${x}`, "stats.json"), "utf8"),
|
||
) as Array<{
|
||
study: string;
|
||
accounts: { rounds: Array<{ round: number; adoptedScore: number | null }> };
|
||
}>;
|
||
return {
|
||
name: stats[0]!.study,
|
||
scores: stats[0]!.accounts.rounds.map((r) => r.adoptedScore),
|
||
};
|
||
};
|
||
function referenceTrajectory(study: string) {
|
||
const db = new DatabaseSync(
|
||
`/home/iomgaa/rb-acceptance/${study}/board.sqlite`,
|
||
{ readOnly: true },
|
||
);
|
||
try {
|
||
const rows = (kind: string) =>
|
||
(
|
||
db
|
||
.prepare("SELECT payload FROM facts WHERE kind = ? ORDER BY created_seq")
|
||
.all(kind) as Array<{ payload: string }>
|
||
).map((r) => JSON.parse(r.payload));
|
||
const obs = rows("run-observation") as Array<{
|
||
runId: string;
|
||
finalMetric: { value: number | null };
|
||
}>;
|
||
return (
|
||
rows("adoption") as Array<{ adoptedRunId: string | null }>
|
||
).map((a) =>
|
||
a.adoptedRunId === null
|
||
? null
|
||
: obs.find((o) => o.runId === a.adoptedRunId)!.finalMetric.value,
|
||
);
|
||
} finally {
|
||
db.close();
|
||
}
|
||
}
|
||
|
||
const summaries = dirs.map((dir) => {
|
||
const studyDir = join(dir, "study");
|
||
const records = readLedger(studyDir);
|
||
const config = frozenConfig(records);
|
||
const accounts = diagnosisAccounts(records);
|
||
const study = config.studyId;
|
||
const studyLabels = labels[study] ?? {};
|
||
|
||
// 轻量检查：实现/修复/诊断实现调用用了几次，有没有超过配置上限。
|
||
const lightChecks = ofKind(records, "callStarted")
|
||
.filter((c) =>
|
||
[
|
||
"implementation",
|
||
"repair",
|
||
"diagnosisImplementation",
|
||
"diagnosisRepair",
|
||
].includes((c.content.materials as TaskMaterial).kind),
|
||
)
|
||
.map((c) => {
|
||
const sessions = join(studyDir, "calls", c.content.call, "sessions");
|
||
return {
|
||
call: c.content.call,
|
||
seq: c.seq,
|
||
...(existsSync(sessions)
|
||
? lightCheckUsage(sessions, config.lightChecks.maxCount)
|
||
: {
|
||
maxCount: config.lightChecks.maxCount,
|
||
pass: false,
|
||
note: "会话目录不存在",
|
||
}),
|
||
};
|
||
});
|
||
|
||
// CPU 时间 / 墙钟时间之比，并附同时段机器负载。
|
||
const timeline = existsSync(join(dir, "timeline.jsonl"))
|
||
? readTimeline(join(dir, "timeline.jsonl"))
|
||
: [];
|
||
const atOf = (seq: number) => timeline.find((t) => t.seq === seq)?.at ?? null;
|
||
const machine = existsSync(join(dir, "machine.jsonl"))
|
||
? readFileSync(join(dir, "machine.jsonl"), "utf8")
|
||
.split("\n")
|
||
.filter(Boolean)
|
||
.map(
|
||
(line) =>
|
||
JSON.parse(line) as { at: string; label: string; load?: number[] },
|
||
)
|
||
: [];
|
||
const ratios = accounts.runs.map((r) => {
|
||
const result = r.results[0] ?? null;
|
||
const ratio = result === null ? null : cpuRatio(result.content);
|
||
const start = r.starts[0] === undefined ? null : atOf(r.starts[0].seq);
|
||
const end = result === null ? null : atOf(result.seq);
|
||
const samples =
|
||
start === null || end === null
|
||
? []
|
||
: machine
|
||
.filter(
|
||
(m) =>
|
||
m.label === "sample" &&
|
||
Date.parse(m.at) >= start &&
|
||
Date.parse(m.at) <= end,
|
||
)
|
||
.map((m) => ({ at: m.at, load: m.load }));
|
||
return {
|
||
run: r.plan.content.run,
|
||
purpose: r.plan.content.purpose,
|
||
round: r.plan.content.round,
|
||
ratio,
|
||
flagged: ratio !== null && ratio < CPU_RATIO_FLAG,
|
||
start: start === null ? null : new Date(start).toISOString(),
|
||
end: end === null ? null : new Date(end).toISOString(),
|
||
meanLoad1:
|
||
samples.length === 0
|
||
? null
|
||
: samples.reduce((s, x) => s + x.load![0]!, 0) / samples.length,
|
||
};
|
||
});
|
||
const measuredRatios = ratios.filter((r) => r.ratio !== null);
|
||
|
||
const killPath = join(dir, "kill-report.json");
|
||
const kill = existsSync(killPath)
|
||
? (JSON.parse(readFileSync(killPath, "utf8")) as {
|
||
evidence: unknown;
|
||
checks: Record<string, boolean>;
|
||
})
|
||
: null;
|
||
|
||
// 第 4 样：每轮一行。
|
||
const dr = accounts.diagnosisRounds;
|
||
const table4 = dr.map((r) => {
|
||
const d = r.direction;
|
||
const label = studyLabels[String(r.round)];
|
||
const proposals = board4(records, r.round);
|
||
const proposalLabels = proposals.map((p) => label?.proposals[p.id] ?? null);
|
||
const candidateRows = r.forward.map((c) => ({
|
||
...c,
|
||
chosen: true,
|
||
label: label?.candidates[c.candidate] ?? null,
|
||
}));
|
||
const along = candidateRows.filter((c) => c.label === "along");
|
||
const diagGain = d === null ? null : d.gain;
|
||
return {
|
||
round: r.round,
|
||
direction:
|
||
d === null
|
||
? r.diagnosed
|
||
? "诊断未结算"
|
||
: "本轮没有诊断"
|
||
: d.found
|
||
? `${d.experiment}（${d.gains.find((g) => g.experiment === d.experiment)!.measured ? "实测" : "估计"}收益 ${f(d.gain)}）`
|
||
: `没有指出能涨分的方向（最大收益 ${f(d.gain)}，${d.experiment}）`,
|
||
found: d?.found ?? null,
|
||
diagnosisGain: diagGain,
|
||
proposalCount: proposals.length,
|
||
proposalsAlong:
|
||
label === undefined ? null : proposalLabels.filter((x) => x === "along").length,
|
||
proposalsCiteDiagnosis: proposals.filter((p) => p.cites).length,
|
||
candidatesAlong: label === undefined ? null : along.length,
|
||
alongChosen: label === undefined ? null : along.length,
|
||
candidates: candidateRows.map((c) => ({
|
||
candidate: c.candidate,
|
||
label: c.label,
|
||
gain: c.gain,
|
||
vsDiagnosis:
|
||
c.gain === null || diagGain === null
|
||
? null
|
||
: c.gain > diagGain
|
||
? "高于诊断实测"
|
||
: "不高于诊断实测",
|
||
})),
|
||
adopted: accounts.rounds.find((x) => x.round === r.round)?.adopted ?? null,
|
||
// 诊断指出了方向却没有人沿它提（有标注才判）。
|
||
directionNotFollowed:
|
||
label === undefined || d === null || !d.found
|
||
? null
|
||
: proposalLabels.every((x) => x === "not"),
|
||
};
|
||
});
|
||
|
||
const pass =
|
||
Object.values(accounts.diagnosisChecks).every(Boolean) &&
|
||
lightChecks.every((c) => c.pass) &&
|
||
(kill === null || Object.values(kill.checks).every(Boolean));
|
||
return {
|
||
dir,
|
||
study,
|
||
accounts,
|
||
lightChecks,
|
||
ratios,
|
||
cpuRatioMin:
|
||
measuredRatios.length === 0
|
||
? null
|
||
: Math.min(...measuredRatios.map((r) => r.ratio!)),
|
||
kill,
|
||
table4,
|
||
pass,
|
||
};
|
||
});
|
||
|
||
/** 本轮的原提案：编号和是否出现"诊断"字样（机械统计，不是"沿方向"的判断）。 */
|
||
function board4(
|
||
records: ReturnType<typeof readLedger>,
|
||
round: number,
|
||
): Array<{ id: string; cites: boolean }> {
|
||
const stage = `r${round}-proposal`;
|
||
const latest = new Map<string, (typeof records)[number]>();
|
||
for (const r of records) {
|
||
if (r.kind === "submission" && r.content.stageId === stage) {
|
||
latest.set(`${r.content.author}/${r.content.item}`, r);
|
||
}
|
||
}
|
||
return [...latest.values()].map((r) => {
|
||
if (r.kind !== "submission") throw new Error("不可能");
|
||
return {
|
||
id: r.content.item!,
|
||
cites: /诊断|diagnos/i.test(JSON.stringify(r.content.content)),
|
||
};
|
||
});
|
||
}
|
||
|
||
jsonFile(join(output, "stats.json"), summaries);
|
||
|
||
const oldB = oldTrajectory("b");
|
||
const oldD = oldTrajectory("d");
|
||
const refs = ["15-real-1", "15-tight-2", "16-real-1"].map((s) => ({
|
||
name: s,
|
||
scores: referenceTrajectory(s),
|
||
}));
|
||
|
||
const text: string[] = [];
|
||
for (const s of summaries) {
|
||
const a = s.accounts;
|
||
text.push(
|
||
`${"=".repeat(70)}`,
|
||
`研究 ${s.study}（${s.dir}）`,
|
||
`第 7 样 程序账目`,
|
||
`确认启动：基线 ${a.confirmedStarts.baseline}、诊断 ${a.runs.filter((r) => r.plan.content.purpose === "diagnosis" && r.starts.length > 0).length}、正向 ${a.confirmedStarts.forward}、消融 ${a.confirmedStarts.ablation}，共 ${a.confirmedStarts.total}；扣次 ${a.balance.charged}，余额 ${a.balance.balance}，占用 ${a.balance.reserved}；已知最好 ${f(a.knownBest)}`,
|
||
`停下：${a.halt === null ? "（没有停下事件）" : a.halt.reason}`,
|
||
`账目核对：${Object.entries(a.diagnosisChecks)
|
||
.map(([k, v]) => `${k}=${v ? "过" : "不过"}`)
|
||
.join("、")}`,
|
||
`轻量检查：${s.lightChecks.every((c) => c.pass) ? "全部在上限内" : "有超限或会话缺失，见 stats.json"}`,
|
||
`CPU/墙钟比：${s.ratios.filter((r) => r.ratio !== null).length} 次运行有比值，最小 ${f(s.cpuRatioMin, 3)}；低于 ${CPU_RATIO_FLAG} 的：${s.ratios.filter((r) => r.flagged).map((r) => `${r.run}=${r.ratio!.toFixed(3)}（同时段 1 分钟负载均值 ${f(r.meanLoad1, 1)}）`).join("、") || "没有"}`,
|
||
s.kill === null
|
||
? "杀进程：这份没做"
|
||
: `杀进程：${Object.entries(s.kill.checks).map(([k, v]) => `${k}=${v ? "过" : "不过"}`).join("、")}`,
|
||
"",
|
||
"每次诊断：轮 | 余额(开轮前) | 诊断 | 选中实验 | 核验次 | 修复次 | 拿掉 | 跑了 | 扣次 | 预测冻结早于启动 | 结算数",
|
||
);
|
||
for (const r of a.diagnosisRounds) {
|
||
text.push(
|
||
`${r.round} | ${r.balanceBefore} | ${r.expectedDiagnosis ? (r.diagnosed ? "做" : r.selectDecided ? "不做" : "还没到选实验") : "无（余额<2）"} | ${r.chosen.join(",") || "—"} | ${r.verificationCount} | ${r.repairCount} | ${r.droppedCount} | ${r.ranCount} | ${r.charged} | ${r.predictionBeforeStart === null ? "—" : r.predictionBeforeStart ? "是" : "否"} | ${r.settlementCount}`,
|
||
);
|
||
}
|
||
text.push(
|
||
"",
|
||
"每个实验：轮 | 实验 | 作者 | 动哪处 | 选中 | 结局 | 实测 | 改进量 | 用时秒 | ending | 世界比特 | 惊讶比特 | 没做原因/拿掉原因",
|
||
);
|
||
for (const r of a.diagnosisRounds) {
|
||
for (const x of r.experiments) {
|
||
text.push(
|
||
`${r.round} | ${x.id} ${x.name} | ${x.authors.join("+")} | ${x.spot} | ${x.chosen ? "是" : "否"} | ${x.outcome ?? "—"} | ${f(x.score)} | ${f(x.gain)} | ${f(x.wallSeconds, 2)} | ${x.ending ?? "—"} | ${f(x.worldBits, 4)} | ${f(x.surprise, 4)} | ${x.dropped ?? (x.skipReason === null ? "—" : `${x.skipReason}（含它最好组合净值 ${e3(x.skipBestNet)}）`)}`,
|
||
);
|
||
}
|
||
}
|
||
|
||
text.push("", "第 2 样 选实验：每轮合并实验、事前期望收益、各组合价值/成本/净值、选了谁");
|
||
for (const r of a.diagnosisRounds) {
|
||
if (!r.expectedDiagnosis) {
|
||
text.push(`第 ${r.round} 轮：余额 ${r.balanceBefore} < 2，无诊断记录`);
|
||
continue;
|
||
}
|
||
text.push(
|
||
`第 ${r.round} 轮（基线 ${f(r.baselineScore)}，参数 ${JSON.stringify(r.selectParams)}，ĝ₀ ${f(r.g0)}）：${r.diagnosed ? `做，选 ${r.chosen.join("、")}，净值 ${e3(r.chosenNet)}` : "不做（所有组合净值都不大于 1e-12）"}`,
|
||
...r.experiments.map(
|
||
(x) =>
|
||
` ${x.id} ${x.name}：事前期望收益 ${f(x.priorGain)}；事前合并范围 ${range(x.merged)}`,
|
||
),
|
||
...r.combos.map(
|
||
(c) =>
|
||
` 组合 {${c.experiments.join(",")}}：VOI ${e3(c.voi)}，价值 ${e3(c.value)}，成本 ${e3(c.cost)}，净值 ${e3(c.net)}（R̂ ${c.rHat}，m′ ${f(c.mPrime, 4)}）`,
|
||
),
|
||
);
|
||
}
|
||
|
||
text.push("", "第 4 样 诊断有没有改变走向（'沿方向'列需子 Agent 标注 labels.json，没有就写 —）");
|
||
text.push("轮 | 诊断推出的方向 | 本轮提案数 | 提案引用'诊断'字样 | 沿方向的提案 | 沿方向的候选(选批进了几个) | 采纳 | 各入选候选：标注/正向改进量/对比诊断实测");
|
||
for (const t of s.table4) {
|
||
text.push(
|
||
`${t.round} | ${t.direction} | ${t.proposalCount} | ${t.proposalsCiteDiagnosis} | ${t.proposalsAlong ?? "—"} | ${t.candidatesAlong ?? "—"} | ${t.adopted ?? "没有"} | ${t.candidates.map((c) => `${c.candidate}:${c.label ?? "—"}/${f(c.gain)}/${c.vsDiagnosis ?? "—"}`).join("；")}`,
|
||
);
|
||
}
|
||
text.push(
|
||
`诊断指出方向却无人沿它提的轮次：${
|
||
s.table4.some((t) => t.directionNotFollowed === null && t.found === true)
|
||
? "待标注"
|
||
: s.table4.filter((t) => t.directionNotFollowed).map((t) => t.round).join("、") || "没有"
|
||
}`,
|
||
);
|
||
|
||
text.push("", "第 5 样 预测准不准、分不分得开");
|
||
for (const r of a.diagnosisRounds) {
|
||
if (r.predictionRows.length === 0 && r.mergedRows.length === 0) {
|
||
text.push(`第 ${r.round} 轮：没有有效实验可对`);
|
||
continue;
|
||
}
|
||
const byAuthor = new Map<string, typeof r.predictionRows>();
|
||
for (const p of r.predictionRows) {
|
||
byAuthor.set(p.author, [...(byAuthor.get(p.author) ?? []), p]);
|
||
}
|
||
text.push(`第 ${r.round} 轮：`);
|
||
for (const [author, rows] of byAuthor) {
|
||
const bands = { "<q1": 0, "[q1,q5)": 0, "[q5,q9]": 0, ">q9": 0 } as Record<string, number>;
|
||
for (const p of rows) bands[p.band] = bands[p.band]! + 1;
|
||
text.push(
|
||
` ${author}：${rows.length} 对，落点 ${JSON.stringify(bands)}，覆盖率 ${rows.filter((p) => p.covered).length}/${rows.length}`,
|
||
);
|
||
}
|
||
const mb = { "<q1": 0, "[q1,q5)": 0, "[q5,q9]": 0, ">q9": 0 } as Record<string, number>;
|
||
for (const p of r.mergedRows) mb[p.band] = mb[p.band]! + 1;
|
||
text.push(
|
||
` 合并：${r.mergedRows.length} 个实验，落点 ${JSON.stringify(mb)}，覆盖率 ${r.mergedRows.filter((p) => p.covered).length}/${r.mergedRows.length}`,
|
||
` 同轮各实验合并中位数差距：${r.experimentGap === null ? "—" : `最大减最小 ${f(r.experimentGap.maxMinusMin)}，相邻最小间隔 ${f(r.experimentGap.minPairwise)}`}；同轮正向候选合并中位数差距：${r.candidateGap === null ? "—" : `最大减最小 ${f(r.candidateGap.maxMinusMin)}，相邻最小间隔 ${f(r.candidateGap.minPairwise)}`}`,
|
||
);
|
||
}
|
||
|
||
text.push("", "第 6 样 逐轮走向与候选");
|
||
for (const r of a.rounds) {
|
||
const dRound = a.diagnosisRounds.find((x) => x.round === r.round)!;
|
||
text.push(
|
||
`第 ${r.round} 轮（基线 ${r.baseline}）：诊断${dRound.diagnosed ? `做了，${dRound.ranCount} 个实验` : dRound.expectedDiagnosis ? (dRound.selectDecided ? "没做（净值不大于 0）" : "还没到选实验") : "没有（余额<2）"}；采纳 ${r.adopted ?? "没有"}${r.adoptedScore === null ? "" : `，分数 ${f(r.adoptedScore)}，离已知最好 ${f(r.knownBestGap)}`}`,
|
||
...r.candidates.map(
|
||
(c) =>
|
||
` 候选 ${c.candidate}：实测 ${f(c.score)}，合并 ${range(c.merged)}，落点 ${c.band ?? "—"}，作者自评 ${c.authorEstimates.map((u) => `${u.author} ${range(u.range)}`).join("、") || "—"}${c.notRunEvents.length > 0 ? "，不跑：" + c.notRunEvents[0]!.reason : ""}`,
|
||
),
|
||
);
|
||
}
|
||
const curve = a.rounds.map((r) => r.adoptedScore);
|
||
text.push(
|
||
`采纳分数曲线 ${s.study}：${curve.map((x) => f(x)).join(" → ")}`,
|
||
`并排：${oldB.name}（无诊断、消融开）${oldB.scores.map((x) => f(x)).join(" → ")}；${oldD.name} ${oldD.scores.map((x) => f(x)).join(" → ")}；${refs.map((x) => `${x.name} ${x.scores.map((y) => f(y)).join(" → ")}`).join("；")}`,
|
||
"",
|
||
);
|
||
}
|
||
writeFileSync(join(output, "stats.txt"), `${text.join("\n")}\n`);
|
||
console.log(
|
||
`统计：${output}；${summaries.length} 份研究的逐研究账目检查${summaries.every((s) => s.pass) ? "通过" : "未通过"}。研究质量判断仍待独立审阅。`,
|
||
);
|
||
if (!summaries.every((s) => s.pass)) process.exitCode = 1;
|