eval: 完整研究的运行、统计与审阅导出（任务书 17 第三块的检验代码）
- eval/full/run.ts：配置照 eval/round-zero/research.json 只换编号，真实跑到停下； launches/machine/timeline/outcome 照任务书 16 惯例；--kill-after-first-ablation-start 在第一个消融运行确认启动后、结果落账前 SIGKILL 控制进程，恢复续跑到底， kill-report.json 写三条检查（同一容器不重跑、只扣一次、结果照常写入）。 - eval/lib/full.ts：账目核对 fullAccounts——逐轮走向（候选分数、采纳、选格、每格 核验重做次数与扣次、结算、押的因子）加 12 项程序账目检查（阶段开关成对、 扣次=确认启动数、余额为 0、解释冻结早于首格启动、每轮最多一条结算、解封在 收尾关闭之后且覆盖全部选中候选、跑不起来/不跑的格零扣次、第 2 轮起基线链 重算、停在"导出还没写"等）与 CPU/墙钟比（<0.9 列出并附同时段负载，只列事实）。 - eval/full/stats.ts、export.ts：统计输出与按轮按格的审阅材料导出。 - 不沿用 eval/forward/acceptance.ts 的"启动区间互不重叠"（这次 3 份同时跑）。 - tests/full-eval.test.ts 4 条：真实账本 + rules 真实计算走完一轮含消融的研究， 撑几成与配合关系标签按方法.md 6.6 手算对照。npm run check 通过，npm test 476 条通过。
This commit is contained in:
@@ -0,0 +1,180 @@
|
||||
import { existsSync, mkdirSync, readFileSync, writeFileSync } from "node:fs";
|
||||
import { join, resolve } from "node:path";
|
||||
import type { TaskMaterial } from "../../src/materials/index.ts";
|
||||
import { cpuRatio, CPU_RATIO_FLAG, fullAccounts } from "../lib/full.ts";
|
||||
import { frozenConfig, jsonFile } from "../lib/forward.ts";
|
||||
import { ofKind, readLedger } from "../lib/ledger.ts";
|
||||
import { lightCheckUsage } from "../lib/light-checks.ts";
|
||||
import { readTimeline } from "../lib/timeline.ts";
|
||||
|
||||
// 任务书 17 第 3.2 节第 6、7 样的可计算部分：逐轮走向 + 程序账目 + CPU/墙钟比。只算事实、不作质量判断
|
||||
// （解释质量、核验判得对不对、收尾与接力由父会话派全新上下文的子 Agent 读 eval/full/export.ts 的产物）。
|
||||
// 这次三份研究同时跑，没有"启动区间互不重叠"这条（任务书 17 第 3.1 节）；也不设总体 acceptance。
|
||||
// node eval/full/stats.ts <全新绝对统计输出目录> <研究输出目录>...
|
||||
const [output, ...dirs] = process.argv.slice(2);
|
||||
if (!output || dirs.length === 0 || output !== resolve(output)) {
|
||||
throw new Error(
|
||||
"用法：node eval/full/stats.ts <全新绝对统计输出目录> <研究输出目录>...",
|
||||
);
|
||||
}
|
||||
mkdirSync(output);
|
||||
|
||||
const fmt = (v: number | null) => (v === null ? "—" : v.toFixed(6));
|
||||
|
||||
const summaries = dirs.map((dir) => {
|
||||
const studyDir = join(dir, "study");
|
||||
const records = readLedger(studyDir);
|
||||
const config = frozenConfig(records);
|
||||
const accounts = fullAccounts(records);
|
||||
|
||||
// 轻量检查：每次实现/修复/消融实现调用用了几次、有没有超过配置上限（任务书 17 第 3.2 节第 7 样）。
|
||||
const lightChecks = ofKind(records, "callStarted")
|
||||
.filter((c) =>
|
||||
["implementation", "repair", "ablationImplementation"].includes(
|
||||
(c.content.materials as TaskMaterial).kind,
|
||||
),
|
||||
)
|
||||
.map((c) => {
|
||||
const sessions = join(studyDir, "calls", c.content.call, "sessions");
|
||||
return {
|
||||
call: c.content.call,
|
||||
seq: c.seq,
|
||||
...(existsSync(sessions)
|
||||
? lightCheckUsage(sessions, config.lightChecks.maxCount)
|
||||
: {
|
||||
maxCount: config.lightChecks.maxCount,
|
||||
pass: false,
|
||||
note: "会话目录不存在",
|
||||
}),
|
||||
};
|
||||
});
|
||||
|
||||
// 每次正式运行的 CPU 时间/墙钟时间之比；比值明显低于 1 的列出，并和 machine.jsonl 同时段负载放在一起（只列事实）。
|
||||
const timeline = existsSync(join(dir, "timeline.jsonl"))
|
||||
? readTimeline(join(dir, "timeline.jsonl"))
|
||||
: [];
|
||||
const atOf = (seq: number) => timeline.find((t) => t.seq === seq)?.at ?? null;
|
||||
const machine = existsSync(join(dir, "machine.jsonl"))
|
||||
? readFileSync(join(dir, "machine.jsonl"), "utf8")
|
||||
.split("\n")
|
||||
.filter(Boolean)
|
||||
.map(
|
||||
(line) =>
|
||||
JSON.parse(line) as { at: string; label: string; load?: number[] },
|
||||
)
|
||||
: [];
|
||||
const ratios = accounts.runs.map((r) => {
|
||||
const result = r.results[0] ?? null;
|
||||
const ratio = result === null ? null : cpuRatio(result.content);
|
||||
const start = r.starts[0] === undefined ? null : atOf(r.starts[0].seq);
|
||||
const end = result === null ? null : atOf(result.seq);
|
||||
const samples =
|
||||
start === null || end === null
|
||||
? []
|
||||
: machine
|
||||
.filter(
|
||||
(m) =>
|
||||
m.label === "sample" &&
|
||||
Date.parse(m.at) >= start &&
|
||||
Date.parse(m.at) <= end,
|
||||
)
|
||||
.map((m) => ({ at: m.at, load: m.load }));
|
||||
return {
|
||||
run: r.plan.content.run,
|
||||
purpose: r.plan.content.purpose,
|
||||
round: r.plan.content.round,
|
||||
object: r.plan.content.object,
|
||||
ratio,
|
||||
flagged: ratio !== null && ratio < CPU_RATIO_FLAG,
|
||||
start: start === null ? null : new Date(start).toISOString(),
|
||||
end: end === null ? null : new Date(end).toISOString(),
|
||||
loadSamples: samples,
|
||||
note: `比值低于 ${CPU_RATIO_FLAG} 列出并附同时段负载；只列事实，不作判断`,
|
||||
};
|
||||
});
|
||||
|
||||
// 有 kill-report.json 的研究，把三条检查的结果抄进来（任务书 17 第 3.1 节）。
|
||||
const kill = existsSync(join(dir, "kill-report.json"))
|
||||
? (JSON.parse(readFileSync(join(dir, "kill-report.json"), "utf8")) as {
|
||||
evidence: unknown;
|
||||
checks: Record<string, boolean>;
|
||||
})
|
||||
: null;
|
||||
|
||||
const pass =
|
||||
Object.values(accounts.checks).every(Boolean) &&
|
||||
lightChecks.every((c) => c.pass) &&
|
||||
(kill === null || Object.values(kill.checks).every(Boolean));
|
||||
return {
|
||||
dir,
|
||||
study: config.studyId,
|
||||
accounts,
|
||||
lightChecks,
|
||||
ratios,
|
||||
kill,
|
||||
pass,
|
||||
};
|
||||
});
|
||||
|
||||
jsonFile(join(output, "stats.json"), summaries);
|
||||
|
||||
const text = summaries.map((s) => {
|
||||
const a = s.accounts;
|
||||
const lines = [
|
||||
`研究 ${s.study}（${s.dir}）`,
|
||||
`确认启动：基线 ${a.confirmedStarts.baseline}、正向 ${a.confirmedStarts.forward}、消融 ${a.confirmedStarts.ablation}，共 ${a.confirmedStarts.total}；扣次 ${a.balance.charged}，余额 ${a.balance.balance}，占用 ${a.balance.reserved}；已知最好 ${fmt(a.knownBest)}`,
|
||||
`停下：${a.halt === null ? "（没有停下事件）" : a.halt.reason}`,
|
||||
`账目核对：${Object.entries(a.checks)
|
||||
.map(([k, v]) => `${k}=${v ? "过" : "不过"}`)
|
||||
.join("、")}`,
|
||||
`轻量检查：${s.lightChecks.every((c) => c.pass) ? "全部在上限内" : "有超限或会话缺失，见 stats.json"}`,
|
||||
`CPU/墙钟比低于 ${CPU_RATIO_FLAG} 的运行：${
|
||||
s.ratios
|
||||
.filter((r) => r.flagged)
|
||||
.map((r) => `${r.run}=${r.ratio!.toFixed(3)}`)
|
||||
.join("、") || "没有"
|
||||
}`,
|
||||
s.kill === null
|
||||
? "杀进程：这份没做"
|
||||
: `杀进程：${Object.entries(s.kill.checks)
|
||||
.map(([k, v]) => `${k}=${v ? "过" : "不过"}`)
|
||||
.join("、")}`,
|
||||
"",
|
||||
];
|
||||
for (const r of a.rounds) {
|
||||
lines.push(
|
||||
`第 ${r.round} 轮（基线 ${r.baseline}）`,
|
||||
...r.candidates.map(
|
||||
(c) =>
|
||||
` 候选 ${c.candidate}：实测 ${fmt(c.score)}，合并 ${c.merged === null ? "—" : `${fmt(c.merged.q1)}/${fmt(c.merged.q5)}/${fmt(c.merged.q9)}`}，落点 ${c.band ?? "—"}，作者自评 ${c.authorEstimates.map((u) => `${u.author} ${fmt(u.range.q1)}/${fmt(u.range.q5)}/${fmt(u.range.q9)}`).join("、") || "—"}${c.notRunEvents.length > 0 ? "，不跑：" + c.notRunEvents[0]!.reason : ""}`,
|
||||
),
|
||||
` 采纳：${r.adopted ?? "没有"}${r.adoptedScore === null ? "" : `，分数 ${fmt(r.adoptedScore)}，离已知最好 ${fmt(r.knownBestGap)}`}`,
|
||||
);
|
||||
if (r.hasAblation) {
|
||||
lines.push(
|
||||
` 消融：选格 ${r.cellsDecision!.output.chosen.join("、") || "（一格都没选）"}；配合关系 ${r.label === null ? "—" : r.label.label}；押的因子 ${r.bet === null ? "—" : `${r.bet.decision === "bet" ? r.bet.factor : `不押（${r.bet.decision}）`}，预测收益 ${fmt(r.bet.predictedGain)}`}`,
|
||||
...r.cells.map(
|
||||
(c) =>
|
||||
` ${c.cell}：核验 ${c.verificationCount} 次、重做 ${c.redoCount} 次${c.cannotRun === null ? "" : "，跑不起来：" + c.cannotRun}${c.notRunEvents.length > 0 ? "，不跑：" + c.notRunEvents[0]!.reason : ""}，跑没跑 ${c.ran ? "跑了" : "没跑"}（扣次 ${c.charged}），实测 ${fmt(c.measured)}，撑几成 ${c.support === null ? "—" : c.support.toFixed(3)}，合并 ${c.merged === null ? "—" : `${fmt(c.merged.q1)}/${fmt(c.merged.q5)}/${fmt(c.merged.q9)}`}，结局 ${c.outcome ?? "—"}`,
|
||||
),
|
||||
...r
|
||||
.cellsDecision!.output.cells.filter((c) => !c.tested)
|
||||
.map(
|
||||
(c) =>
|
||||
` ${c.factor} 没测：${c.reason}（含它的最好组合净值 ${c.bestCombo === null ? "没有可选组合" : c.bestCombo.net.toExponential(3)}）`,
|
||||
),
|
||||
);
|
||||
} else lines.push(" 消融：这一轮没有");
|
||||
lines.push(
|
||||
` 解释关闭早于首格启动：${r.explanationBeforeCells === null ? "无消融" : r.explanationBeforeCells ? "是" : "否"}；解封在收尾关闭之后且覆盖选中候选：${r.unseal.pass ? "是" : "否"}`,
|
||||
);
|
||||
lines.push("");
|
||||
}
|
||||
return lines.join("\n");
|
||||
});
|
||||
writeFileSync(join(output, "stats.txt"), `${text.join("\n")}\n`);
|
||||
|
||||
console.log(
|
||||
`统计：${output}；所给 ${summaries.length} 份研究的逐研究账目检查${summaries.every((s) => s.pass) ? "通过" : "未通过"}。不设总体 acceptance（三份同时跑，无互不重叠条款）；研究质量判断仍待独立审阅。`,
|
||||
);
|
||||
if (!summaries.every((s) => s.pass)) process.exitCode = 1;
|
||||
Reference in New Issue
Block a user