eval: 完整研究的运行、统计与审阅导出（任务书 17 第三块的检验代码）
- eval/full/run.ts：配置照 eval/round-zero/research.json 只换编号，真实跑到停下； launches/machine/timeline/outcome 照任务书 16 惯例；--kill-after-first-ablation-start 在第一个消融运行确认启动后、结果落账前 SIGKILL 控制进程，恢复续跑到底， kill-report.json 写三条检查（同一容器不重跑、只扣一次、结果照常写入）。 - eval/lib/full.ts：账目核对 fullAccounts——逐轮走向（候选分数、采纳、选格、每格 核验重做次数与扣次、结算、押的因子）加 12 项程序账目检查（阶段开关成对、 扣次=确认启动数、余额为 0、解释冻结早于首格启动、每轮最多一条结算、解封在 收尾关闭之后且覆盖全部选中候选、跑不起来/不跑的格零扣次、第 2 轮起基线链 重算、停在"导出还没写"等）与 CPU/墙钟比（<0.9 列出并附同时段负载，只列事实）。 - eval/full/stats.ts、export.ts：统计输出与按轮按格的审阅材料导出。 - 不沿用 eval/forward/acceptance.ts 的"启动区间互不重叠"（这次 3 份同时跑）。 - tests/full-eval.test.ts 4 条：真实账本 + rules 真实计算走完一轮含消融的研究， 撑几成与配合关系标签按方法.md 6.6 手算对照。npm run check 通过，npm test 476 条通过。
This commit is contained in:
@@ -0,0 +1,273 @@
|
||||
import { mkdirSync, writeFileSync } from "node:fs";
|
||||
import { join, resolve } from "node:path";
|
||||
import {
|
||||
Board,
|
||||
cellId,
|
||||
explanationId,
|
||||
protocolMetrics,
|
||||
splitCell,
|
||||
type TaskMaterial,
|
||||
} from "../../src/materials/index.ts";
|
||||
import { ablationOutcome } from "../../src/submissions/index.ts";
|
||||
import { checkoutVersion, compareVersions } from "../../src/workspace/index.ts";
|
||||
import { jsonFile } from "../lib/forward.ts";
|
||||
import { finalSubmissions, ofKind, readLedger } from "../lib/ledger.ts";
|
||||
|
||||
// 任务书 17 第 3.2 节研究质量评估的审阅材料：解释、选格决定、消融格（协议、位置表、代码与 diff、实现与核验往来）、
|
||||
// 结算、收尾、下一轮提案。只导出事实和原文，不替审阅者下结论。
|
||||
// node eval/full/export.ts <研究输出目录> <全新绝对审阅目录>
|
||||
const [dir, output, ...extra] = process.argv.slice(2);
|
||||
if (!dir || !output || extra.length || output !== resolve(output)) {
|
||||
throw new Error(
|
||||
"用法：node eval/full/export.ts <研究输出目录> <全新绝对审阅目录>",
|
||||
);
|
||||
}
|
||||
mkdirSync(output); // 不覆盖以前的审阅材料。
|
||||
const study = join(dir, "study");
|
||||
const records = readLedger(study);
|
||||
const board = new Board(records);
|
||||
const repo = join(study, "repo.git");
|
||||
const finals = finalSubmissions(records);
|
||||
const roundOf = (stageId: string) => board.stage(stageId).round;
|
||||
const roundIds = [
|
||||
...new Set(ofKind(records, "decision").map((d) => d.content.round)),
|
||||
]
|
||||
.filter((r) => r >= 1)
|
||||
.sort((a, b) => a - b);
|
||||
|
||||
const checkout = (version: string, path: string) => {
|
||||
mkdirSync(path);
|
||||
checkoutVersion(repo, version, path);
|
||||
};
|
||||
const json = (v: unknown) => `${JSON.stringify(v, null, 2)}\n`;
|
||||
|
||||
const index = [];
|
||||
for (const round of roundIds) {
|
||||
const roundDir = join(output, `round-${round}`);
|
||||
mkdirSync(roundDir);
|
||||
|
||||
// 本轮提案（下一轮提案在 round-<r+1> 里，审阅"收尾→接力"时把两份对着读）。
|
||||
const proposals = finals.filter(
|
||||
(s) =>
|
||||
s.content.kind === "proposal" &&
|
||||
roundOf(s.content.stageId) === round,
|
||||
);
|
||||
jsonFile(join(roundDir, "proposals.json"), proposals);
|
||||
writeFileSync(
|
||||
join(roundDir, "proposals.txt"),
|
||||
proposals
|
||||
.map(
|
||||
(s) =>
|
||||
`${"=".repeat(20)} ${s.content.item}（${s.content.author}，记录 ${s.seq}）\n${json(s.content.content)}`,
|
||||
)
|
||||
.join("\n"),
|
||||
);
|
||||
|
||||
// 解释（任务书 17 第 3.2 节第 1 样）：作者、正文、每格三个数。
|
||||
const explanations = finals.filter(
|
||||
(s) =>
|
||||
s.content.kind === "explanation" &&
|
||||
roundOf(s.content.stageId) === round,
|
||||
);
|
||||
jsonFile(join(roundDir, "explanations.json"), explanations);
|
||||
writeFileSync(
|
||||
join(roundDir, "explanations.txt"),
|
||||
explanations
|
||||
.flatMap((s) =>
|
||||
(
|
||||
s.content.content as {
|
||||
explanations: Array<{
|
||||
body: string;
|
||||
cells?: Array<{ factor: string; range: unknown }>;
|
||||
}>;
|
||||
}
|
||||
).explanations.map(
|
||||
(e, i) =>
|
||||
`${"=".repeat(20)} ${explanationId(round, s.content.author, i + 1)}（${s.content.author} 的第 ${i + 1} 条，记录 ${s.seq}）\n${e.body}\n三个数：${(e.cells ?? []).map((c) => `${c.factor} ${JSON.stringify(c.range)}`).join("；")}`,
|
||||
),
|
||||
)
|
||||
.join("\n\n"),
|
||||
);
|
||||
|
||||
// 选格决定全文（第 2 样）：因子、合并范围、各组合的信息价值/成本/净值、选中与没选的原因都在这条决定里。
|
||||
const cellsDecision =
|
||||
ofKind(records, "decision").find(
|
||||
(d) => d.content.decision === "cells" && d.content.round === round,
|
||||
) ?? null;
|
||||
const chosenCells = board.decision("cells", round)?.output.chosen ?? null;
|
||||
jsonFile(join(roundDir, "cells-decision.json"), cellsDecision);
|
||||
|
||||
// 结算全文（第 4 样）：结算记录和同一事务里押的因子决定。
|
||||
const settlement =
|
||||
ofKind(records, "settlement").find((s) => s.content.round === round) ??
|
||||
null;
|
||||
const bet =
|
||||
ofKind(records, "decision").find(
|
||||
(d) => d.content.decision === "bet" && d.content.round === round,
|
||||
) ?? null;
|
||||
jsonFile(join(roundDir, "settlement.json"), { settlement, bet });
|
||||
|
||||
// 收尾全文（第 5 样）。
|
||||
const wrapUps = finals.filter(
|
||||
(s) =>
|
||||
s.content.kind === "wrapUp" && roundOf(s.content.stageId) === round,
|
||||
);
|
||||
jsonFile(join(roundDir, "wrapups.json"), wrapUps);
|
||||
writeFileSync(
|
||||
join(roundDir, "wrapups.txt"),
|
||||
wrapUps
|
||||
.map(
|
||||
(s) =>
|
||||
`${"=".repeat(20)} ${s.content.author}（记录 ${s.seq}）\n${(s.content.content as { body: string }).body}`,
|
||||
)
|
||||
.join("\n\n"),
|
||||
);
|
||||
|
||||
// 消融格（第 3 样）：冻结协议、位置表、完整代码与格代码及 diff、alsoOff、concerns、每次核验逐项结论与退回、重做记录。
|
||||
const cells = [];
|
||||
if (chosenCells !== null) {
|
||||
const adopt = board.decision("adopt", round)!;
|
||||
const candidate = adopt.output.adopted!;
|
||||
const full = board.code(candidate);
|
||||
const baseline = board.code(board.baselineOf(round));
|
||||
for (const key of chosenCells) {
|
||||
const cell = cellId(candidate, key);
|
||||
const cellDir = join(roundDir, "cells", cell);
|
||||
mkdirSync(cellDir, { recursive: true });
|
||||
jsonFile(join(cellDir, "protocol.json"), {
|
||||
record: ofKind(records, "protocol")
|
||||
.filter((p) => p.content.candidate === candidate)
|
||||
.at(-1),
|
||||
resolvedMetrics: protocolMetrics(
|
||||
board,
|
||||
board.protocol(candidate),
|
||||
),
|
||||
factorLocations: full.locations,
|
||||
baselineVersion: baseline.version,
|
||||
fullVersion: full.version,
|
||||
});
|
||||
checkout(full.version, join(cellDir, "full"));
|
||||
checkout(baseline.version, join(cellDir, "baseline"));
|
||||
// 每次实现和代码版本对号（同一事务里先写提交、紧接着写代码；交"跑不起来"时没有代码记录）。
|
||||
const impls = finals.filter(
|
||||
(s) =>
|
||||
s.content.kind === "ablationImplementation" &&
|
||||
s.content.target === cell,
|
||||
);
|
||||
const codes = ofKind(records, "code").filter(
|
||||
(c) => c.content.object === cell,
|
||||
);
|
||||
const attempts = impls.map((s) => ({
|
||||
submission: s,
|
||||
code: codes.find((c) => c.seq === s.seq + 1) ?? null,
|
||||
}));
|
||||
jsonFile(join(cellDir, "implementations.json"), attempts);
|
||||
const verifications = finals
|
||||
.filter(
|
||||
(s) =>
|
||||
s.content.kind === "ablationVerification" &&
|
||||
s.content.target === cell,
|
||||
)
|
||||
.map((s) => {
|
||||
const impl =
|
||||
impls.filter((x) => x.seq < s.seq).at(-1) ?? null;
|
||||
return {
|
||||
submission: s,
|
||||
judgedImplSeq: impl?.seq ?? null,
|
||||
outcome:
|
||||
impl === null
|
||||
? null
|
||||
: ablationOutcome(
|
||||
s.content.content as Parameters<
|
||||
typeof ablationOutcome
|
||||
>[0],
|
||||
key,
|
||||
(
|
||||
impl.content.content as {
|
||||
alsoOff: string[];
|
||||
}
|
||||
).alsoOff,
|
||||
),
|
||||
};
|
||||
});
|
||||
jsonFile(join(cellDir, "verifications.json"), verifications);
|
||||
jsonFile(
|
||||
join(cellDir, "rejections.json"),
|
||||
ofKind(records, "rejection").filter(
|
||||
(r) => r.content.target === cell,
|
||||
),
|
||||
);
|
||||
const lastCode = codes.at(-1) ?? null;
|
||||
if (lastCode !== null) {
|
||||
checkout(lastCode.content.version, join(cellDir, "cell"));
|
||||
jsonFile(
|
||||
join(cellDir, "diff.json"),
|
||||
compareVersions(
|
||||
repo,
|
||||
full.version,
|
||||
lastCode.content.version,
|
||||
),
|
||||
);
|
||||
}
|
||||
for (const attempt of attempts) {
|
||||
if (attempt.code === null) continue;
|
||||
const path = join(cellDir, `code-${attempt.code.seq}`);
|
||||
checkout(attempt.code.content.version, path);
|
||||
jsonFile(
|
||||
join(cellDir, `diff-${attempt.code.seq}.json`),
|
||||
compareVersions(
|
||||
repo,
|
||||
full.version,
|
||||
attempt.code.content.version,
|
||||
),
|
||||
);
|
||||
}
|
||||
jsonFile(
|
||||
join(cellDir, "calls.json"),
|
||||
ofKind(records, "callStarted")
|
||||
.filter((c) => {
|
||||
const material = c.content.materials as TaskMaterial;
|
||||
return (
|
||||
material.object === cell &&
|
||||
[
|
||||
"ablationImplementation",
|
||||
"ablationVerification",
|
||||
].includes(material.kind)
|
||||
);
|
||||
})
|
||||
.map((c) => ({
|
||||
record: c,
|
||||
sessions: join(
|
||||
study,
|
||||
"calls",
|
||||
c.content.call,
|
||||
"sessions",
|
||||
),
|
||||
})),
|
||||
);
|
||||
cells.push({
|
||||
cell,
|
||||
factor: splitCell(cell).key,
|
||||
dir: cellDir,
|
||||
ran: lastCode !== null,
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
writeFileSync(
|
||||
join(roundDir, "review.txt"),
|
||||
`任务书17第3.2节，只读本目录，中文直接回复，不写文件。\n` +
|
||||
`第1样（解释）：读 explanations.txt、cells-decision.json、settlement.json。看不到消融结果地回答：机制是否具体、写了什么情况下不成立没有、三个数和正文是否一致、范围宽窄有没有依据、同轮几人的解释有没有真正不同、"没有具体机制判断"用得诚不诚实；再对 settlement.json 看每条解释在每格被确认还是否定。\n` +
|
||||
`第2样（选格）：读 cells-decision.json，列出各格合并范围、各组合信息价值/成本/净值、选中和没选的原因。\n` +
|
||||
`第3样（消融实现与核验，每格一个全新上下文）：读 cells/<格>/ 下的 protocol.json、full/、cell/、diff.json、implementations.json、verifications.json、rejections.json。判断：是不是只关了指定的因子；保障原样保留没有、关掉的因子的指标写 null 没有；答案路径标签和关掉以后的分支对不对得上；核验员每次逐项结论判得对不对；重做的只改了被指出的地方没有。\n` +
|
||||
`第5样（收尾与接力）：读 wrapups.txt、settlement.json，再对着 round-${round + 1}/proposals.txt（没有就是最后一轮）看下一轮提案用上了哪些发现。\n`,
|
||||
);
|
||||
index.push({
|
||||
round,
|
||||
dir: roundDir,
|
||||
hasAblation: chosenCells !== null,
|
||||
cells,
|
||||
});
|
||||
}
|
||||
jsonFile(join(output, "index.json"), { study: dir, rounds: index });
|
||||
console.log(`审阅材料：${output}`);
|
||||
@@ -0,0 +1,325 @@
|
||||
import { execFileSync } from "node:child_process";
|
||||
import {
|
||||
appendFileSync,
|
||||
existsSync,
|
||||
mkdirSync,
|
||||
readFileSync,
|
||||
writeFileSync,
|
||||
} from "node:fs";
|
||||
import { loadavg, freemem, totalmem } from "node:os";
|
||||
import { join, resolve } from "node:path";
|
||||
import { fullAccounts, FULL_STOP } from "../lib/full.ts";
|
||||
import { jsonFile } from "../lib/forward.ts";
|
||||
import {
|
||||
sameContainerThroughResult,
|
||||
type ContainerSnapshot,
|
||||
} from "../lib/forward-evidence.ts";
|
||||
import { ofKind, readLedger } from "../lib/ledger.ts";
|
||||
import { launch, sleep, waitForLedger, writeConfig } from "../lib/study.ts";
|
||||
import { watchTimeline } from "../lib/timeline.ts";
|
||||
|
||||
// 任务书 17 第 3.1 节：从头真实跑一份完整研究，到余额用完停在"导出还没写"。三份同时跑，由父会话各起一个进程；
|
||||
// 容器名带研究编号，这里只认本研究的，不和别的研究互相核对。
|
||||
// node eval/full/run.ts <研究编号> <全新绝对输出目录> [--kill-after-first-ablation-start]
|
||||
// 服务一时调不通等中断后的续跑：npm run eval:study -- --resume --dir <输出目录>/study（照任务书 16 的做法）。
|
||||
|
||||
const [name, output, ...extra] = process.argv.slice(2);
|
||||
const kill =
|
||||
extra.length === 1 && extra[0] === "--kill-after-first-ablation-start";
|
||||
if (
|
||||
!name ||
|
||||
!output ||
|
||||
extra.length > 1 ||
|
||||
(extra.length === 1 && !kill) ||
|
||||
output !== resolve(output)
|
||||
) {
|
||||
throw new Error(
|
||||
"用法：node eval/full/run.ts <研究编号> <全新绝对输出目录> [--kill-after-first-ablation-start]",
|
||||
);
|
||||
}
|
||||
const out = output;
|
||||
if (existsSync(out)) throw new Error(`输出目录已存在，不覆盖现场：${out}`);
|
||||
mkdirSync(out, { recursive: true });
|
||||
const studyDir = join(out, "study");
|
||||
const config = writeConfig(
|
||||
"eval/round-zero/research.json",
|
||||
join(out, "research.json"),
|
||||
name,
|
||||
null,
|
||||
);
|
||||
|
||||
const daemonTime = () =>
|
||||
execFileSync("docker", ["info", "--format", "{{.SystemTime}}"], {
|
||||
encoding: "utf8",
|
||||
}).trim();
|
||||
const inspect = (
|
||||
container: string,
|
||||
): ContainerSnapshot & {
|
||||
Name: string;
|
||||
Created: string;
|
||||
State: {
|
||||
Running: boolean;
|
||||
StartedAt: string;
|
||||
FinishedAt: string;
|
||||
Status: string;
|
||||
};
|
||||
} =>
|
||||
JSON.parse(
|
||||
execFileSync("docker", ["inspect", container], { encoding: "utf8" }),
|
||||
)[0];
|
||||
|
||||
/** 事件点的重快照（进程列表、容器列表），照任务书 16 的 machine.jsonl 做法。 */
|
||||
function observe(label: string) {
|
||||
appendFileSync(
|
||||
join(out, "machine.jsonl"),
|
||||
`${JSON.stringify({
|
||||
at: new Date().toISOString(),
|
||||
label,
|
||||
processes: execFileSync("ps", ["-eo", "pid,ppid,lstart,args"], {
|
||||
encoding: "utf8",
|
||||
}),
|
||||
containers: execFileSync("docker", ["ps", "--no-trunc"], {
|
||||
encoding: "utf8",
|
||||
}),
|
||||
})}\n`,
|
||||
);
|
||||
}
|
||||
|
||||
/** 运行期间的轻量负载采样，几秒一条（任务书 17 第 3.1 节：和每次运行的 CPU/墙钟比对照，只列事实）。 */
|
||||
function sample() {
|
||||
appendFileSync(
|
||||
join(out, "machine.jsonl"),
|
||||
`${JSON.stringify({ at: new Date().toISOString(), label: "sample", load: loadavg(), memory: { total: totalmem(), free: freemem() } })}\n`,
|
||||
);
|
||||
}
|
||||
|
||||
function start(path: string | null) {
|
||||
observe(path === null ? "resume" : "start");
|
||||
const study = launch(path, studyDir, {});
|
||||
appendFileSync(
|
||||
join(out, "launches.jsonl"),
|
||||
`${JSON.stringify({ event: "start", pid: study.child.pid, at: new Date().toISOString(), config: path })}\n`,
|
||||
);
|
||||
const interrupt = () => study.child.kill("SIGINT");
|
||||
process.on("SIGINT", interrupt);
|
||||
study.done.then((end) => {
|
||||
process.off("SIGINT", interrupt);
|
||||
appendFileSync(
|
||||
join(out, "launches.jsonl"),
|
||||
`${JSON.stringify({ event: "end", pid: study.child.pid, at: new Date().toISOString(), ...end })}\n`,
|
||||
);
|
||||
});
|
||||
return study;
|
||||
}
|
||||
|
||||
const sampler = setInterval(sample, 5000);
|
||||
sampler.unref();
|
||||
let active = start(config);
|
||||
let killEvidence: {
|
||||
run: string;
|
||||
cell: string;
|
||||
round: number;
|
||||
killSeq: number;
|
||||
} | null = null;
|
||||
try {
|
||||
await waitForLedger(studyDir, active.child);
|
||||
let watching = watchTimeline(
|
||||
studyDir,
|
||||
active.child,
|
||||
join(out, "timeline.jsonl"),
|
||||
);
|
||||
if (kill) {
|
||||
// 盯住账本：第一个用途为 ablation 的运行确认启动之后、结果落账之前杀控制进程（照 eval/forward/run.ts 的 kill 做法）。
|
||||
let hit = false;
|
||||
while (active.child.exitCode === null && active.child.signalCode === null) {
|
||||
const records = readLedger(studyDir);
|
||||
const plan = ofKind(records, "runPlanned").find(
|
||||
(r) => r.content.purpose === "ablation",
|
||||
);
|
||||
if (plan !== undefined) {
|
||||
const run = plan.content.run;
|
||||
if (ofKind(records, "runResult").some((r) => r.content.run === run)) {
|
||||
throw new Error(
|
||||
"第一个消融运行已经有结果了，未打中；保留现场，不改杀别的格",
|
||||
);
|
||||
}
|
||||
const started = ofKind(records, "runStarted").find(
|
||||
(r) => r.content.run === run,
|
||||
);
|
||||
if (started !== undefined) {
|
||||
// 容器名是正式入口写在结构文档里的约定（inforsi-<研究编号>-<运行编号>），不 import flow 内部文件。
|
||||
const container = `inforsi-${name}-${run}`;
|
||||
// 从杀前划定历史事件区间，覆盖恢复启动到目标结果落账；不把 spawn 当成恢复已执行。
|
||||
const lifecycleSince = daemonTime();
|
||||
const before = inspect(container);
|
||||
jsonFile(join(out, "container-before.json"), before);
|
||||
if (!before.State.Running)
|
||||
throw new Error("第一个消融容器已不在运行，未打中");
|
||||
jsonFile(join(out, "kill.json"), {
|
||||
run,
|
||||
cell: plan.content.object,
|
||||
round: plan.content.round,
|
||||
planSeq: plan.seq,
|
||||
startSeq: started.seq,
|
||||
beforeKillSeq: records.at(-1)!.seq,
|
||||
pid: active.child.pid,
|
||||
at: new Date().toISOString(),
|
||||
container,
|
||||
});
|
||||
if (!active.child.kill("SIGKILL")) throw new Error("未发出 SIGKILL");
|
||||
const killed = await active.done;
|
||||
await watching;
|
||||
if (killed.signal !== "SIGKILL")
|
||||
throw new Error("控制进程不是以 SIGKILL 结束");
|
||||
const atKill = readLedger(studyDir);
|
||||
jsonFile(join(out, "ledger-at-kill.json"), atKill);
|
||||
if (ofKind(atKill, "runResult").some((r) => r.content.run === run)) {
|
||||
throw new Error("杀后核对发现目标运行已经写结果，未打中；保留现场");
|
||||
}
|
||||
jsonFile(join(out, "container-after-kill.json"), inspect(container));
|
||||
observe("after-kill");
|
||||
jsonFile(
|
||||
join(out, "container-before-resume.json"),
|
||||
inspect(container),
|
||||
);
|
||||
killEvidence = {
|
||||
run,
|
||||
cell: plan.content.object!,
|
||||
round: plan.content.round,
|
||||
killSeq: atKill.at(-1)!.seq,
|
||||
};
|
||||
active = start(null);
|
||||
watching = watchTimeline(
|
||||
studyDir,
|
||||
active.child,
|
||||
join(out, "timeline.jsonl"),
|
||||
);
|
||||
let targetResult;
|
||||
while (true) {
|
||||
targetResult = ofKind(readLedger(studyDir), "runResult").find(
|
||||
(r) => r.content.run === run,
|
||||
);
|
||||
if (
|
||||
targetResult !== undefined ||
|
||||
active.child.exitCode !== null ||
|
||||
active.child.signalCode !== null
|
||||
)
|
||||
break;
|
||||
await sleep(100);
|
||||
}
|
||||
const lifecycleUntil = daemonTime();
|
||||
// Docker CLI 按时间回读，不依赖事件订阅的启动时序；原文全存，stats 核对缓存上限和目标 die。
|
||||
writeFileSync(
|
||||
join(out, "container-events.jsonl"),
|
||||
execFileSync(
|
||||
"docker",
|
||||
[
|
||||
"events",
|
||||
"--since",
|
||||
lifecycleSince,
|
||||
"--until",
|
||||
lifecycleUntil,
|
||||
"--format",
|
||||
"{{json .}}",
|
||||
],
|
||||
{ encoding: "utf8" },
|
||||
),
|
||||
);
|
||||
jsonFile(join(out, "container-events-window.json"), {
|
||||
since: lifecycleSince,
|
||||
until: lifecycleUntil,
|
||||
run,
|
||||
resultSeq: targetResult?.seq ?? null,
|
||||
});
|
||||
if (targetResult === undefined)
|
||||
throw new Error(
|
||||
"恢复进程没有交出目标结果，生命周期证据不完整；保留现场",
|
||||
);
|
||||
hit = true;
|
||||
break;
|
||||
}
|
||||
}
|
||||
await sleep(100);
|
||||
}
|
||||
if (!hit)
|
||||
throw new Error(
|
||||
`没有打中第一个消融运行：${JSON.stringify(await active.done)}`,
|
||||
);
|
||||
}
|
||||
await watching;
|
||||
const end = await active.done;
|
||||
jsonFile(join(out, "outcome.json"), end);
|
||||
observe("end");
|
||||
const records = readLedger(studyDir);
|
||||
const accounts = fullAccounts(records);
|
||||
jsonFile(join(out, "accounts.json"), accounts);
|
||||
|
||||
// 杀进程的三条检查（任务书 17 第 3.1 节）：①恢复按容器名接着等同一个容器、没有重跑；②只扣一次；③结果照常写入、结算和后续步骤走完。
|
||||
if (killEvidence !== null) {
|
||||
const evidence = JSON.parse(
|
||||
readFileSync(join(out, "kill.json"), "utf8"),
|
||||
) as { run: string; cell: string; round: number; container: string };
|
||||
const snapshots = (["before", "after-kill", "before-resume"] as const).map(
|
||||
(part) =>
|
||||
JSON.parse(
|
||||
readFileSync(join(out, `container-${part}.json`), "utf8"),
|
||||
) as ContainerSnapshot,
|
||||
);
|
||||
const lifecycle = readFileSync(join(out, "container-events.jsonl"), "utf8")
|
||||
.split("\n")
|
||||
.filter(Boolean)
|
||||
.map((line) => JSON.parse(line));
|
||||
const lifecycleChecks = sameContainerThroughResult(
|
||||
snapshots,
|
||||
lifecycle,
|
||||
evidence.container,
|
||||
);
|
||||
const target = accounts.runs.find(
|
||||
(r) => r.plan.content.run === evidence.run,
|
||||
)!;
|
||||
const cellPlans = accounts.runs.filter(
|
||||
(r) =>
|
||||
r.plan.content.purpose === "ablation" &&
|
||||
r.plan.content.object === evidence.cell,
|
||||
);
|
||||
const report = {
|
||||
evidence,
|
||||
killSeq: killEvidence.killSeq,
|
||||
snapshotMeaning: "杀前、杀后、恢复启动前；没有把快照当成恢复完成的证据",
|
||||
lifecycleChecks,
|
||||
checks: {
|
||||
sameContainerNoRerun:
|
||||
Object.values(lifecycleChecks).every(Boolean) &&
|
||||
cellPlans.length === 1 &&
|
||||
target.starts.length === 1,
|
||||
chargedOnce: target.starts.length === 1,
|
||||
resultSettledAndFinished:
|
||||
target.results.length === 1 &&
|
||||
target.results[0]!.seq > killEvidence.killSeq &&
|
||||
ofKind(records, "settlement").some(
|
||||
(s) => s.content.round === evidence.round,
|
||||
) &&
|
||||
accounts.halt !== null &&
|
||||
accounts.halt.reason === FULL_STOP,
|
||||
},
|
||||
};
|
||||
jsonFile(join(out, "kill-report.json"), report);
|
||||
}
|
||||
|
||||
if (
|
||||
end.code !== 0 ||
|
||||
!accounts.checks.haltedAtExport ||
|
||||
!accounts.checks.chargedEqualsStarts
|
||||
) {
|
||||
throw new Error(
|
||||
`没有走到"导出还没写"或账目不平，保留现场；查明原因后用 npm run eval:study -- --resume --dir ${studyDir} 续跑：${JSON.stringify(end)}`,
|
||||
);
|
||||
}
|
||||
console.log(`研究走完：${studyDir}。仍需 export、stats 和独立质量审阅。`);
|
||||
} finally {
|
||||
clearInterval(sampler);
|
||||
if (active.child.exitCode === null && active.child.signalCode === null) {
|
||||
active.child.kill("SIGINT");
|
||||
await active.done;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,180 @@
|
||||
import { existsSync, mkdirSync, readFileSync, writeFileSync } from "node:fs";
|
||||
import { join, resolve } from "node:path";
|
||||
import type { TaskMaterial } from "../../src/materials/index.ts";
|
||||
import { cpuRatio, CPU_RATIO_FLAG, fullAccounts } from "../lib/full.ts";
|
||||
import { frozenConfig, jsonFile } from "../lib/forward.ts";
|
||||
import { ofKind, readLedger } from "../lib/ledger.ts";
|
||||
import { lightCheckUsage } from "../lib/light-checks.ts";
|
||||
import { readTimeline } from "../lib/timeline.ts";
|
||||
|
||||
// 任务书 17 第 3.2 节第 6、7 样的可计算部分：逐轮走向 + 程序账目 + CPU/墙钟比。只算事实、不作质量判断
|
||||
// （解释质量、核验判得对不对、收尾与接力由父会话派全新上下文的子 Agent 读 eval/full/export.ts 的产物）。
|
||||
// 这次三份研究同时跑，没有"启动区间互不重叠"这条（任务书 17 第 3.1 节）；也不设总体 acceptance。
|
||||
// node eval/full/stats.ts <全新绝对统计输出目录> <研究输出目录>...
|
||||
const [output, ...dirs] = process.argv.slice(2);
|
||||
if (!output || dirs.length === 0 || output !== resolve(output)) {
|
||||
throw new Error(
|
||||
"用法：node eval/full/stats.ts <全新绝对统计输出目录> <研究输出目录>...",
|
||||
);
|
||||
}
|
||||
mkdirSync(output);
|
||||
|
||||
const fmt = (v: number | null) => (v === null ? "—" : v.toFixed(6));
|
||||
|
||||
const summaries = dirs.map((dir) => {
|
||||
const studyDir = join(dir, "study");
|
||||
const records = readLedger(studyDir);
|
||||
const config = frozenConfig(records);
|
||||
const accounts = fullAccounts(records);
|
||||
|
||||
// 轻量检查：每次实现/修复/消融实现调用用了几次、有没有超过配置上限（任务书 17 第 3.2 节第 7 样）。
|
||||
const lightChecks = ofKind(records, "callStarted")
|
||||
.filter((c) =>
|
||||
["implementation", "repair", "ablationImplementation"].includes(
|
||||
(c.content.materials as TaskMaterial).kind,
|
||||
),
|
||||
)
|
||||
.map((c) => {
|
||||
const sessions = join(studyDir, "calls", c.content.call, "sessions");
|
||||
return {
|
||||
call: c.content.call,
|
||||
seq: c.seq,
|
||||
...(existsSync(sessions)
|
||||
? lightCheckUsage(sessions, config.lightChecks.maxCount)
|
||||
: {
|
||||
maxCount: config.lightChecks.maxCount,
|
||||
pass: false,
|
||||
note: "会话目录不存在",
|
||||
}),
|
||||
};
|
||||
});
|
||||
|
||||
// 每次正式运行的 CPU 时间/墙钟时间之比；比值明显低于 1 的列出，并和 machine.jsonl 同时段负载放在一起（只列事实）。
|
||||
const timeline = existsSync(join(dir, "timeline.jsonl"))
|
||||
? readTimeline(join(dir, "timeline.jsonl"))
|
||||
: [];
|
||||
const atOf = (seq: number) => timeline.find((t) => t.seq === seq)?.at ?? null;
|
||||
const machine = existsSync(join(dir, "machine.jsonl"))
|
||||
? readFileSync(join(dir, "machine.jsonl"), "utf8")
|
||||
.split("\n")
|
||||
.filter(Boolean)
|
||||
.map(
|
||||
(line) =>
|
||||
JSON.parse(line) as { at: string; label: string; load?: number[] },
|
||||
)
|
||||
: [];
|
||||
const ratios = accounts.runs.map((r) => {
|
||||
const result = r.results[0] ?? null;
|
||||
const ratio = result === null ? null : cpuRatio(result.content);
|
||||
const start = r.starts[0] === undefined ? null : atOf(r.starts[0].seq);
|
||||
const end = result === null ? null : atOf(result.seq);
|
||||
const samples =
|
||||
start === null || end === null
|
||||
? []
|
||||
: machine
|
||||
.filter(
|
||||
(m) =>
|
||||
m.label === "sample" &&
|
||||
Date.parse(m.at) >= start &&
|
||||
Date.parse(m.at) <= end,
|
||||
)
|
||||
.map((m) => ({ at: m.at, load: m.load }));
|
||||
return {
|
||||
run: r.plan.content.run,
|
||||
purpose: r.plan.content.purpose,
|
||||
round: r.plan.content.round,
|
||||
object: r.plan.content.object,
|
||||
ratio,
|
||||
flagged: ratio !== null && ratio < CPU_RATIO_FLAG,
|
||||
start: start === null ? null : new Date(start).toISOString(),
|
||||
end: end === null ? null : new Date(end).toISOString(),
|
||||
loadSamples: samples,
|
||||
note: `比值低于 ${CPU_RATIO_FLAG} 列出并附同时段负载；只列事实，不作判断`,
|
||||
};
|
||||
});
|
||||
|
||||
// 有 kill-report.json 的研究，把三条检查的结果抄进来（任务书 17 第 3.1 节）。
|
||||
const kill = existsSync(join(dir, "kill-report.json"))
|
||||
? (JSON.parse(readFileSync(join(dir, "kill-report.json"), "utf8")) as {
|
||||
evidence: unknown;
|
||||
checks: Record<string, boolean>;
|
||||
})
|
||||
: null;
|
||||
|
||||
const pass =
|
||||
Object.values(accounts.checks).every(Boolean) &&
|
||||
lightChecks.every((c) => c.pass) &&
|
||||
(kill === null || Object.values(kill.checks).every(Boolean));
|
||||
return {
|
||||
dir,
|
||||
study: config.studyId,
|
||||
accounts,
|
||||
lightChecks,
|
||||
ratios,
|
||||
kill,
|
||||
pass,
|
||||
};
|
||||
});
|
||||
|
||||
jsonFile(join(output, "stats.json"), summaries);
|
||||
|
||||
const text = summaries.map((s) => {
|
||||
const a = s.accounts;
|
||||
const lines = [
|
||||
`研究 ${s.study}（${s.dir}）`,
|
||||
`确认启动：基线 ${a.confirmedStarts.baseline}、正向 ${a.confirmedStarts.forward}、消融 ${a.confirmedStarts.ablation}，共 ${a.confirmedStarts.total}；扣次 ${a.balance.charged}，余额 ${a.balance.balance}，占用 ${a.balance.reserved}；已知最好 ${fmt(a.knownBest)}`,
|
||||
`停下：${a.halt === null ? "（没有停下事件）" : a.halt.reason}`,
|
||||
`账目核对：${Object.entries(a.checks)
|
||||
.map(([k, v]) => `${k}=${v ? "过" : "不过"}`)
|
||||
.join("、")}`,
|
||||
`轻量检查：${s.lightChecks.every((c) => c.pass) ? "全部在上限内" : "有超限或会话缺失，见 stats.json"}`,
|
||||
`CPU/墙钟比低于 ${CPU_RATIO_FLAG} 的运行：${
|
||||
s.ratios
|
||||
.filter((r) => r.flagged)
|
||||
.map((r) => `${r.run}=${r.ratio!.toFixed(3)}`)
|
||||
.join("、") || "没有"
|
||||
}`,
|
||||
s.kill === null
|
||||
? "杀进程：这份没做"
|
||||
: `杀进程：${Object.entries(s.kill.checks)
|
||||
.map(([k, v]) => `${k}=${v ? "过" : "不过"}`)
|
||||
.join("、")}`,
|
||||
"",
|
||||
];
|
||||
for (const r of a.rounds) {
|
||||
lines.push(
|
||||
`第 ${r.round} 轮（基线 ${r.baseline}）`,
|
||||
...r.candidates.map(
|
||||
(c) =>
|
||||
` 候选 ${c.candidate}：实测 ${fmt(c.score)}，合并 ${c.merged === null ? "—" : `${fmt(c.merged.q1)}/${fmt(c.merged.q5)}/${fmt(c.merged.q9)}`}，落点 ${c.band ?? "—"}，作者自评 ${c.authorEstimates.map((u) => `${u.author} ${fmt(u.range.q1)}/${fmt(u.range.q5)}/${fmt(u.range.q9)}`).join("、") || "—"}${c.notRunEvents.length > 0 ? "，不跑：" + c.notRunEvents[0]!.reason : ""}`,
|
||||
),
|
||||
` 采纳：${r.adopted ?? "没有"}${r.adoptedScore === null ? "" : `，分数 ${fmt(r.adoptedScore)}，离已知最好 ${fmt(r.knownBestGap)}`}`,
|
||||
);
|
||||
if (r.hasAblation) {
|
||||
lines.push(
|
||||
` 消融：选格 ${r.cellsDecision!.output.chosen.join("、") || "（一格都没选）"}；配合关系 ${r.label === null ? "—" : r.label.label}；押的因子 ${r.bet === null ? "—" : `${r.bet.decision === "bet" ? r.bet.factor : `不押（${r.bet.decision}）`}，预测收益 ${fmt(r.bet.predictedGain)}`}`,
|
||||
...r.cells.map(
|
||||
(c) =>
|
||||
` ${c.cell}：核验 ${c.verificationCount} 次、重做 ${c.redoCount} 次${c.cannotRun === null ? "" : "，跑不起来：" + c.cannotRun}${c.notRunEvents.length > 0 ? "，不跑：" + c.notRunEvents[0]!.reason : ""}，跑没跑 ${c.ran ? "跑了" : "没跑"}（扣次 ${c.charged}），实测 ${fmt(c.measured)}，撑几成 ${c.support === null ? "—" : c.support.toFixed(3)}，合并 ${c.merged === null ? "—" : `${fmt(c.merged.q1)}/${fmt(c.merged.q5)}/${fmt(c.merged.q9)}`}，结局 ${c.outcome ?? "—"}`,
|
||||
),
|
||||
...r
|
||||
.cellsDecision!.output.cells.filter((c) => !c.tested)
|
||||
.map(
|
||||
(c) =>
|
||||
` ${c.factor} 没测：${c.reason}（含它的最好组合净值 ${c.bestCombo === null ? "没有可选组合" : c.bestCombo.net.toExponential(3)}）`,
|
||||
),
|
||||
);
|
||||
} else lines.push(" 消融：这一轮没有");
|
||||
lines.push(
|
||||
` 解释关闭早于首格启动：${r.explanationBeforeCells === null ? "无消融" : r.explanationBeforeCells ? "是" : "否"}；解封在收尾关闭之后且覆盖选中候选：${r.unseal.pass ? "是" : "否"}`,
|
||||
);
|
||||
lines.push("");
|
||||
}
|
||||
return lines.join("\n");
|
||||
});
|
||||
writeFileSync(join(output, "stats.txt"), `${text.join("\n")}\n`);
|
||||
|
||||
console.log(
|
||||
`统计：${output}；所给 ${summaries.length} 份研究的逐研究账目检查${summaries.every((s) => s.pass) ? "通过" : "未通过"}。不设总体 acceptance（三份同时跑，无互不重叠条款）；研究质量判断仍待独立审阅。`,
|
||||
);
|
||||
if (!summaries.every((s) => s.pass)) process.exitCode = 1;
|
||||
Reference in New Issue
Block a user