iomgaa
0ae495f8e8
- eval/full/run.ts：配置照 eval/round-zero/research.json 只换编号，真实跑到停下； launches/machine/timeline/outcome 照任务书 16 惯例；--kill-after-first-ablation-start 在第一个消融运行确认启动后、结果落账前 SIGKILL 控制进程，恢复续跑到底， kill-report.json 写三条检查（同一容器不重跑、只扣一次、结果照常写入）。 - eval/lib/full.ts：账目核对 fullAccounts——逐轮走向（候选分数、采纳、选格、每格 核验重做次数与扣次、结算、押的因子）加 12 项程序账目检查（阶段开关成对、 扣次=确认启动数、余额为 0、解释冻结早于首格启动、每轮最多一条结算、解封在 收尾关闭之后且覆盖全部选中候选、跑不起来/不跑的格零扣次、第 2 轮起基线链 重算、停在"导出还没写"等）与 CPU/墙钟比（<0.9 列出并附同时段负载，只列事实）。 - eval/full/stats.ts、export.ts：统计输出与按轮按格的审阅材料导出。 - 不沿用 eval/forward/acceptance.ts 的"启动区间互不重叠"（这次 3 份同时跑）。 - tests/full-eval.test.ts 4 条：真实账本 + rules 真实计算走完一轮含消融的研究， 撑几成与配合关系标签按方法.md 6.6 手算对照。npm run check 通过，npm test 476 条通过。
181 lines
8.4 KiB
TypeScript
181 lines
8.4 KiB
TypeScript
import { existsSync, mkdirSync, readFileSync, writeFileSync } from "node:fs";
|
||
import { join, resolve } from "node:path";
|
||
import type { TaskMaterial } from "../../src/materials/index.ts";
|
||
import { cpuRatio, CPU_RATIO_FLAG, fullAccounts } from "../lib/full.ts";
|
||
import { frozenConfig, jsonFile } from "../lib/forward.ts";
|
||
import { ofKind, readLedger } from "../lib/ledger.ts";
|
||
import { lightCheckUsage } from "../lib/light-checks.ts";
|
||
import { readTimeline } from "../lib/timeline.ts";
|
||
|
||
// 任务书 17 第 3.2 节第 6、7 样的可计算部分：逐轮走向 + 程序账目 + CPU/墙钟比。只算事实、不作质量判断
|
||
// （解释质量、核验判得对不对、收尾与接力由父会话派全新上下文的子 Agent 读 eval/full/export.ts 的产物）。
|
||
// 这次三份研究同时跑，没有"启动区间互不重叠"这条（任务书 17 第 3.1 节）；也不设总体 acceptance。
|
||
// node eval/full/stats.ts <全新绝对统计输出目录> <研究输出目录>...
|
||
const [output, ...dirs] = process.argv.slice(2);
|
||
if (!output || dirs.length === 0 || output !== resolve(output)) {
|
||
throw new Error(
|
||
"用法：node eval/full/stats.ts <全新绝对统计输出目录> <研究输出目录>...",
|
||
);
|
||
}
|
||
mkdirSync(output);
|
||
|
||
const fmt = (v: number | null) => (v === null ? "—" : v.toFixed(6));
|
||
|
||
const summaries = dirs.map((dir) => {
|
||
const studyDir = join(dir, "study");
|
||
const records = readLedger(studyDir);
|
||
const config = frozenConfig(records);
|
||
const accounts = fullAccounts(records);
|
||
|
||
// 轻量检查：每次实现/修复/消融实现调用用了几次、有没有超过配置上限（任务书 17 第 3.2 节第 7 样）。
|
||
const lightChecks = ofKind(records, "callStarted")
|
||
.filter((c) =>
|
||
["implementation", "repair", "ablationImplementation"].includes(
|
||
(c.content.materials as TaskMaterial).kind,
|
||
),
|
||
)
|
||
.map((c) => {
|
||
const sessions = join(studyDir, "calls", c.content.call, "sessions");
|
||
return {
|
||
call: c.content.call,
|
||
seq: c.seq,
|
||
...(existsSync(sessions)
|
||
? lightCheckUsage(sessions, config.lightChecks.maxCount)
|
||
: {
|
||
maxCount: config.lightChecks.maxCount,
|
||
pass: false,
|
||
note: "会话目录不存在",
|
||
}),
|
||
};
|
||
});
|
||
|
||
// 每次正式运行的 CPU 时间/墙钟时间之比；比值明显低于 1 的列出，并和 machine.jsonl 同时段负载放在一起（只列事实）。
|
||
const timeline = existsSync(join(dir, "timeline.jsonl"))
|
||
? readTimeline(join(dir, "timeline.jsonl"))
|
||
: [];
|
||
const atOf = (seq: number) => timeline.find((t) => t.seq === seq)?.at ?? null;
|
||
const machine = existsSync(join(dir, "machine.jsonl"))
|
||
? readFileSync(join(dir, "machine.jsonl"), "utf8")
|
||
.split("\n")
|
||
.filter(Boolean)
|
||
.map(
|
||
(line) =>
|
||
JSON.parse(line) as { at: string; label: string; load?: number[] },
|
||
)
|
||
: [];
|
||
const ratios = accounts.runs.map((r) => {
|
||
const result = r.results[0] ?? null;
|
||
const ratio = result === null ? null : cpuRatio(result.content);
|
||
const start = r.starts[0] === undefined ? null : atOf(r.starts[0].seq);
|
||
const end = result === null ? null : atOf(result.seq);
|
||
const samples =
|
||
start === null || end === null
|
||
? []
|
||
: machine
|
||
.filter(
|
||
(m) =>
|
||
m.label === "sample" &&
|
||
Date.parse(m.at) >= start &&
|
||
Date.parse(m.at) <= end,
|
||
)
|
||
.map((m) => ({ at: m.at, load: m.load }));
|
||
return {
|
||
run: r.plan.content.run,
|
||
purpose: r.plan.content.purpose,
|
||
round: r.plan.content.round,
|
||
object: r.plan.content.object,
|
||
ratio,
|
||
flagged: ratio !== null && ratio < CPU_RATIO_FLAG,
|
||
start: start === null ? null : new Date(start).toISOString(),
|
||
end: end === null ? null : new Date(end).toISOString(),
|
||
loadSamples: samples,
|
||
note: `比值低于 ${CPU_RATIO_FLAG} 列出并附同时段负载；只列事实，不作判断`,
|
||
};
|
||
});
|
||
|
||
// 有 kill-report.json 的研究，把三条检查的结果抄进来（任务书 17 第 3.1 节）。
|
||
const kill = existsSync(join(dir, "kill-report.json"))
|
||
? (JSON.parse(readFileSync(join(dir, "kill-report.json"), "utf8")) as {
|
||
evidence: unknown;
|
||
checks: Record<string, boolean>;
|
||
})
|
||
: null;
|
||
|
||
const pass =
|
||
Object.values(accounts.checks).every(Boolean) &&
|
||
lightChecks.every((c) => c.pass) &&
|
||
(kill === null || Object.values(kill.checks).every(Boolean));
|
||
return {
|
||
dir,
|
||
study: config.studyId,
|
||
accounts,
|
||
lightChecks,
|
||
ratios,
|
||
kill,
|
||
pass,
|
||
};
|
||
});
|
||
|
||
jsonFile(join(output, "stats.json"), summaries);
|
||
|
||
const text = summaries.map((s) => {
|
||
const a = s.accounts;
|
||
const lines = [
|
||
`研究 ${s.study}（${s.dir}）`,
|
||
`确认启动：基线 ${a.confirmedStarts.baseline}、正向 ${a.confirmedStarts.forward}、消融 ${a.confirmedStarts.ablation}，共 ${a.confirmedStarts.total}；扣次 ${a.balance.charged}，余额 ${a.balance.balance}，占用 ${a.balance.reserved}；已知最好 ${fmt(a.knownBest)}`,
|
||
`停下：${a.halt === null ? "（没有停下事件）" : a.halt.reason}`,
|
||
`账目核对：${Object.entries(a.checks)
|
||
.map(([k, v]) => `${k}=${v ? "过" : "不过"}`)
|
||
.join("、")}`,
|
||
`轻量检查：${s.lightChecks.every((c) => c.pass) ? "全部在上限内" : "有超限或会话缺失，见 stats.json"}`,
|
||
`CPU/墙钟比低于 ${CPU_RATIO_FLAG} 的运行：${
|
||
s.ratios
|
||
.filter((r) => r.flagged)
|
||
.map((r) => `${r.run}=${r.ratio!.toFixed(3)}`)
|
||
.join("、") || "没有"
|
||
}`,
|
||
s.kill === null
|
||
? "杀进程：这份没做"
|
||
: `杀进程：${Object.entries(s.kill.checks)
|
||
.map(([k, v]) => `${k}=${v ? "过" : "不过"}`)
|
||
.join("、")}`,
|
||
"",
|
||
];
|
||
for (const r of a.rounds) {
|
||
lines.push(
|
||
`第 ${r.round} 轮（基线 ${r.baseline}）`,
|
||
...r.candidates.map(
|
||
(c) =>
|
||
` 候选 ${c.candidate}：实测 ${fmt(c.score)}，合并 ${c.merged === null ? "—" : `${fmt(c.merged.q1)}/${fmt(c.merged.q5)}/${fmt(c.merged.q9)}`}，落点 ${c.band ?? "—"}，作者自评 ${c.authorEstimates.map((u) => `${u.author} ${fmt(u.range.q1)}/${fmt(u.range.q5)}/${fmt(u.range.q9)}`).join("、") || "—"}${c.notRunEvents.length > 0 ? "，不跑：" + c.notRunEvents[0]!.reason : ""}`,
|
||
),
|
||
` 采纳：${r.adopted ?? "没有"}${r.adoptedScore === null ? "" : `，分数 ${fmt(r.adoptedScore)}，离已知最好 ${fmt(r.knownBestGap)}`}`,
|
||
);
|
||
if (r.hasAblation) {
|
||
lines.push(
|
||
` 消融：选格 ${r.cellsDecision!.output.chosen.join("、") || "（一格都没选）"}；配合关系 ${r.label === null ? "—" : r.label.label}；押的因子 ${r.bet === null ? "—" : `${r.bet.decision === "bet" ? r.bet.factor : `不押（${r.bet.decision}）`}，预测收益 ${fmt(r.bet.predictedGain)}`}`,
|
||
...r.cells.map(
|
||
(c) =>
|
||
` ${c.cell}：核验 ${c.verificationCount} 次、重做 ${c.redoCount} 次${c.cannotRun === null ? "" : "，跑不起来：" + c.cannotRun}${c.notRunEvents.length > 0 ? "，不跑：" + c.notRunEvents[0]!.reason : ""}，跑没跑 ${c.ran ? "跑了" : "没跑"}（扣次 ${c.charged}），实测 ${fmt(c.measured)}，撑几成 ${c.support === null ? "—" : c.support.toFixed(3)}，合并 ${c.merged === null ? "—" : `${fmt(c.merged.q1)}/${fmt(c.merged.q5)}/${fmt(c.merged.q9)}`}，结局 ${c.outcome ?? "—"}`,
|
||
),
|
||
...r
|
||
.cellsDecision!.output.cells.filter((c) => !c.tested)
|
||
.map(
|
||
(c) =>
|
||
` ${c.factor} 没测：${c.reason}（含它的最好组合净值 ${c.bestCombo === null ? "没有可选组合" : c.bestCombo.net.toExponential(3)}）`,
|
||
),
|
||
);
|
||
} else lines.push(" 消融：这一轮没有");
|
||
lines.push(
|
||
` 解释关闭早于首格启动：${r.explanationBeforeCells === null ? "无消融" : r.explanationBeforeCells ? "是" : "否"}；解封在收尾关闭之后且覆盖选中候选：${r.unseal.pass ? "是" : "否"}`,
|
||
);
|
||
lines.push("");
|
||
}
|
||
return lines.join("\n");
|
||
});
|
||
writeFileSync(join(output, "stats.txt"), `${text.join("\n")}\n`);
|
||
|
||
console.log(
|
||
`统计：${output}；所给 ${summaries.length} 份研究的逐研究账目检查${summaries.every((s) => s.pass) ? "通过" : "未通过"}。不设总体 acceptance（三份同时跑，无互不重叠条款）；研究质量判断仍待独立审阅。`,
|
||
);
|
||
if (!summaries.every((s) => s.pass)) process.exitCode = 1;
|