eval: 完整研究的运行、统计与审阅导出（任务书 17 第三块的检验代码）
- eval/full/run.ts：配置照 eval/round-zero/research.json 只换编号，真实跑到停下； launches/machine/timeline/outcome 照任务书 16 惯例；--kill-after-first-ablation-start 在第一个消融运行确认启动后、结果落账前 SIGKILL 控制进程，恢复续跑到底， kill-report.json 写三条检查（同一容器不重跑、只扣一次、结果照常写入）。 - eval/lib/full.ts：账目核对 fullAccounts——逐轮走向（候选分数、采纳、选格、每格 核验重做次数与扣次、结算、押的因子）加 12 项程序账目检查（阶段开关成对、 扣次=确认启动数、余额为 0、解释冻结早于首格启动、每轮最多一条结算、解封在 收尾关闭之后且覆盖全部选中候选、跑不起来/不跑的格零扣次、第 2 轮起基线链 重算、停在"导出还没写"等）与 CPU/墙钟比（<0.9 列出并附同时段负载，只列事实）。 - eval/full/stats.ts、export.ts：统计输出与按轮按格的审阅材料导出。 - 不沿用 eval/forward/acceptance.ts 的"启动区间互不重叠"（这次 3 份同时跑）。 - tests/full-eval.test.ts 4 条：真实账本 + rules 真实计算走完一轮含消融的研究， 撑几成与配合关系标签按方法.md 6.6 手算对照。npm run check 通过，npm test 476 条通过。
This commit is contained in:
@@ -0,0 +1,273 @@
|
|||||||
|
import { mkdirSync, writeFileSync } from "node:fs";
|
||||||
|
import { join, resolve } from "node:path";
|
||||||
|
import {
|
||||||
|
Board,
|
||||||
|
cellId,
|
||||||
|
explanationId,
|
||||||
|
protocolMetrics,
|
||||||
|
splitCell,
|
||||||
|
type TaskMaterial,
|
||||||
|
} from "../../src/materials/index.ts";
|
||||||
|
import { ablationOutcome } from "../../src/submissions/index.ts";
|
||||||
|
import { checkoutVersion, compareVersions } from "../../src/workspace/index.ts";
|
||||||
|
import { jsonFile } from "../lib/forward.ts";
|
||||||
|
import { finalSubmissions, ofKind, readLedger } from "../lib/ledger.ts";
|
||||||
|
|
||||||
|
// 任务书 17 第 3.2 节研究质量评估的审阅材料：解释、选格决定、消融格（协议、位置表、代码与 diff、实现与核验往来）、
|
||||||
|
// 结算、收尾、下一轮提案。只导出事实和原文，不替审阅者下结论。
|
||||||
|
// node eval/full/export.ts <研究输出目录> <全新绝对审阅目录>
|
||||||
|
const [dir, output, ...extra] = process.argv.slice(2);
|
||||||
|
if (!dir || !output || extra.length || output !== resolve(output)) {
|
||||||
|
throw new Error(
|
||||||
|
"用法：node eval/full/export.ts <研究输出目录> <全新绝对审阅目录>",
|
||||||
|
);
|
||||||
|
}
|
||||||
|
mkdirSync(output); // 不覆盖以前的审阅材料。
|
||||||
|
const study = join(dir, "study");
|
||||||
|
const records = readLedger(study);
|
||||||
|
const board = new Board(records);
|
||||||
|
const repo = join(study, "repo.git");
|
||||||
|
const finals = finalSubmissions(records);
|
||||||
|
const roundOf = (stageId: string) => board.stage(stageId).round;
|
||||||
|
const roundIds = [
|
||||||
|
...new Set(ofKind(records, "decision").map((d) => d.content.round)),
|
||||||
|
]
|
||||||
|
.filter((r) => r >= 1)
|
||||||
|
.sort((a, b) => a - b);
|
||||||
|
|
||||||
|
const checkout = (version: string, path: string) => {
|
||||||
|
mkdirSync(path);
|
||||||
|
checkoutVersion(repo, version, path);
|
||||||
|
};
|
||||||
|
const json = (v: unknown) => `${JSON.stringify(v, null, 2)}\n`;
|
||||||
|
|
||||||
|
const index = [];
|
||||||
|
for (const round of roundIds) {
|
||||||
|
const roundDir = join(output, `round-${round}`);
|
||||||
|
mkdirSync(roundDir);
|
||||||
|
|
||||||
|
// 本轮提案（下一轮提案在 round-<r+1> 里，审阅"收尾→接力"时把两份对着读）。
|
||||||
|
const proposals = finals.filter(
|
||||||
|
(s) =>
|
||||||
|
s.content.kind === "proposal" &&
|
||||||
|
roundOf(s.content.stageId) === round,
|
||||||
|
);
|
||||||
|
jsonFile(join(roundDir, "proposals.json"), proposals);
|
||||||
|
writeFileSync(
|
||||||
|
join(roundDir, "proposals.txt"),
|
||||||
|
proposals
|
||||||
|
.map(
|
||||||
|
(s) =>
|
||||||
|
`${"=".repeat(20)} ${s.content.item}（${s.content.author}，记录 ${s.seq}）\n${json(s.content.content)}`,
|
||||||
|
)
|
||||||
|
.join("\n"),
|
||||||
|
);
|
||||||
|
|
||||||
|
// 解释（任务书 17 第 3.2 节第 1 样）：作者、正文、每格三个数。
|
||||||
|
const explanations = finals.filter(
|
||||||
|
(s) =>
|
||||||
|
s.content.kind === "explanation" &&
|
||||||
|
roundOf(s.content.stageId) === round,
|
||||||
|
);
|
||||||
|
jsonFile(join(roundDir, "explanations.json"), explanations);
|
||||||
|
writeFileSync(
|
||||||
|
join(roundDir, "explanations.txt"),
|
||||||
|
explanations
|
||||||
|
.flatMap((s) =>
|
||||||
|
(
|
||||||
|
s.content.content as {
|
||||||
|
explanations: Array<{
|
||||||
|
body: string;
|
||||||
|
cells?: Array<{ factor: string; range: unknown }>;
|
||||||
|
}>;
|
||||||
|
}
|
||||||
|
).explanations.map(
|
||||||
|
(e, i) =>
|
||||||
|
`${"=".repeat(20)} ${explanationId(round, s.content.author, i + 1)}（${s.content.author} 的第 ${i + 1} 条，记录 ${s.seq}）\n${e.body}\n三个数：${(e.cells ?? []).map((c) => `${c.factor} ${JSON.stringify(c.range)}`).join("；")}`,
|
||||||
|
),
|
||||||
|
)
|
||||||
|
.join("\n\n"),
|
||||||
|
);
|
||||||
|
|
||||||
|
// 选格决定全文（第 2 样）：因子、合并范围、各组合的信息价值/成本/净值、选中与没选的原因都在这条决定里。
|
||||||
|
const cellsDecision =
|
||||||
|
ofKind(records, "decision").find(
|
||||||
|
(d) => d.content.decision === "cells" && d.content.round === round,
|
||||||
|
) ?? null;
|
||||||
|
const chosenCells = board.decision("cells", round)?.output.chosen ?? null;
|
||||||
|
jsonFile(join(roundDir, "cells-decision.json"), cellsDecision);
|
||||||
|
|
||||||
|
// 结算全文（第 4 样）：结算记录和同一事务里押的因子决定。
|
||||||
|
const settlement =
|
||||||
|
ofKind(records, "settlement").find((s) => s.content.round === round) ??
|
||||||
|
null;
|
||||||
|
const bet =
|
||||||
|
ofKind(records, "decision").find(
|
||||||
|
(d) => d.content.decision === "bet" && d.content.round === round,
|
||||||
|
) ?? null;
|
||||||
|
jsonFile(join(roundDir, "settlement.json"), { settlement, bet });
|
||||||
|
|
||||||
|
// 收尾全文（第 5 样）。
|
||||||
|
const wrapUps = finals.filter(
|
||||||
|
(s) =>
|
||||||
|
s.content.kind === "wrapUp" && roundOf(s.content.stageId) === round,
|
||||||
|
);
|
||||||
|
jsonFile(join(roundDir, "wrapups.json"), wrapUps);
|
||||||
|
writeFileSync(
|
||||||
|
join(roundDir, "wrapups.txt"),
|
||||||
|
wrapUps
|
||||||
|
.map(
|
||||||
|
(s) =>
|
||||||
|
`${"=".repeat(20)} ${s.content.author}（记录 ${s.seq}）\n${(s.content.content as { body: string }).body}`,
|
||||||
|
)
|
||||||
|
.join("\n\n"),
|
||||||
|
);
|
||||||
|
|
||||||
|
// 消融格（第 3 样）：冻结协议、位置表、完整代码与格代码及 diff、alsoOff、concerns、每次核验逐项结论与退回、重做记录。
|
||||||
|
const cells = [];
|
||||||
|
if (chosenCells !== null) {
|
||||||
|
const adopt = board.decision("adopt", round)!;
|
||||||
|
const candidate = adopt.output.adopted!;
|
||||||
|
const full = board.code(candidate);
|
||||||
|
const baseline = board.code(board.baselineOf(round));
|
||||||
|
for (const key of chosenCells) {
|
||||||
|
const cell = cellId(candidate, key);
|
||||||
|
const cellDir = join(roundDir, "cells", cell);
|
||||||
|
mkdirSync(cellDir, { recursive: true });
|
||||||
|
jsonFile(join(cellDir, "protocol.json"), {
|
||||||
|
record: ofKind(records, "protocol")
|
||||||
|
.filter((p) => p.content.candidate === candidate)
|
||||||
|
.at(-1),
|
||||||
|
resolvedMetrics: protocolMetrics(
|
||||||
|
board,
|
||||||
|
board.protocol(candidate),
|
||||||
|
),
|
||||||
|
factorLocations: full.locations,
|
||||||
|
baselineVersion: baseline.version,
|
||||||
|
fullVersion: full.version,
|
||||||
|
});
|
||||||
|
checkout(full.version, join(cellDir, "full"));
|
||||||
|
checkout(baseline.version, join(cellDir, "baseline"));
|
||||||
|
// 每次实现和代码版本对号（同一事务里先写提交、紧接着写代码；交"跑不起来"时没有代码记录）。
|
||||||
|
const impls = finals.filter(
|
||||||
|
(s) =>
|
||||||
|
s.content.kind === "ablationImplementation" &&
|
||||||
|
s.content.target === cell,
|
||||||
|
);
|
||||||
|
const codes = ofKind(records, "code").filter(
|
||||||
|
(c) => c.content.object === cell,
|
||||||
|
);
|
||||||
|
const attempts = impls.map((s) => ({
|
||||||
|
submission: s,
|
||||||
|
code: codes.find((c) => c.seq === s.seq + 1) ?? null,
|
||||||
|
}));
|
||||||
|
jsonFile(join(cellDir, "implementations.json"), attempts);
|
||||||
|
const verifications = finals
|
||||||
|
.filter(
|
||||||
|
(s) =>
|
||||||
|
s.content.kind === "ablationVerification" &&
|
||||||
|
s.content.target === cell,
|
||||||
|
)
|
||||||
|
.map((s) => {
|
||||||
|
const impl =
|
||||||
|
impls.filter((x) => x.seq < s.seq).at(-1) ?? null;
|
||||||
|
return {
|
||||||
|
submission: s,
|
||||||
|
judgedImplSeq: impl?.seq ?? null,
|
||||||
|
outcome:
|
||||||
|
impl === null
|
||||||
|
? null
|
||||||
|
: ablationOutcome(
|
||||||
|
s.content.content as Parameters<
|
||||||
|
typeof ablationOutcome
|
||||||
|
>[0],
|
||||||
|
key,
|
||||||
|
(
|
||||||
|
impl.content.content as {
|
||||||
|
alsoOff: string[];
|
||||||
|
}
|
||||||
|
).alsoOff,
|
||||||
|
),
|
||||||
|
};
|
||||||
|
});
|
||||||
|
jsonFile(join(cellDir, "verifications.json"), verifications);
|
||||||
|
jsonFile(
|
||||||
|
join(cellDir, "rejections.json"),
|
||||||
|
ofKind(records, "rejection").filter(
|
||||||
|
(r) => r.content.target === cell,
|
||||||
|
),
|
||||||
|
);
|
||||||
|
const lastCode = codes.at(-1) ?? null;
|
||||||
|
if (lastCode !== null) {
|
||||||
|
checkout(lastCode.content.version, join(cellDir, "cell"));
|
||||||
|
jsonFile(
|
||||||
|
join(cellDir, "diff.json"),
|
||||||
|
compareVersions(
|
||||||
|
repo,
|
||||||
|
full.version,
|
||||||
|
lastCode.content.version,
|
||||||
|
),
|
||||||
|
);
|
||||||
|
}
|
||||||
|
for (const attempt of attempts) {
|
||||||
|
if (attempt.code === null) continue;
|
||||||
|
const path = join(cellDir, `code-${attempt.code.seq}`);
|
||||||
|
checkout(attempt.code.content.version, path);
|
||||||
|
jsonFile(
|
||||||
|
join(cellDir, `diff-${attempt.code.seq}.json`),
|
||||||
|
compareVersions(
|
||||||
|
repo,
|
||||||
|
full.version,
|
||||||
|
attempt.code.content.version,
|
||||||
|
),
|
||||||
|
);
|
||||||
|
}
|
||||||
|
jsonFile(
|
||||||
|
join(cellDir, "calls.json"),
|
||||||
|
ofKind(records, "callStarted")
|
||||||
|
.filter((c) => {
|
||||||
|
const material = c.content.materials as TaskMaterial;
|
||||||
|
return (
|
||||||
|
material.object === cell &&
|
||||||
|
[
|
||||||
|
"ablationImplementation",
|
||||||
|
"ablationVerification",
|
||||||
|
].includes(material.kind)
|
||||||
|
);
|
||||||
|
})
|
||||||
|
.map((c) => ({
|
||||||
|
record: c,
|
||||||
|
sessions: join(
|
||||||
|
study,
|
||||||
|
"calls",
|
||||||
|
c.content.call,
|
||||||
|
"sessions",
|
||||||
|
),
|
||||||
|
})),
|
||||||
|
);
|
||||||
|
cells.push({
|
||||||
|
cell,
|
||||||
|
factor: splitCell(cell).key,
|
||||||
|
dir: cellDir,
|
||||||
|
ran: lastCode !== null,
|
||||||
|
});
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
writeFileSync(
|
||||||
|
join(roundDir, "review.txt"),
|
||||||
|
`任务书17第3.2节，只读本目录，中文直接回复，不写文件。\n` +
|
||||||
|
`第1样（解释）：读 explanations.txt、cells-decision.json、settlement.json。看不到消融结果地回答：机制是否具体、写了什么情况下不成立没有、三个数和正文是否一致、范围宽窄有没有依据、同轮几人的解释有没有真正不同、"没有具体机制判断"用得诚不诚实；再对 settlement.json 看每条解释在每格被确认还是否定。\n` +
|
||||||
|
`第2样（选格）：读 cells-decision.json，列出各格合并范围、各组合信息价值/成本/净值、选中和没选的原因。\n` +
|
||||||
|
`第3样（消融实现与核验，每格一个全新上下文）：读 cells/<格>/ 下的 protocol.json、full/、cell/、diff.json、implementations.json、verifications.json、rejections.json。判断：是不是只关了指定的因子；保障原样保留没有、关掉的因子的指标写 null 没有；答案路径标签和关掉以后的分支对不对得上；核验员每次逐项结论判得对不对；重做的只改了被指出的地方没有。\n` +
|
||||||
|
`第5样（收尾与接力）：读 wrapups.txt、settlement.json，再对着 round-${round + 1}/proposals.txt（没有就是最后一轮）看下一轮提案用上了哪些发现。\n`,
|
||||||
|
);
|
||||||
|
index.push({
|
||||||
|
round,
|
||||||
|
dir: roundDir,
|
||||||
|
hasAblation: chosenCells !== null,
|
||||||
|
cells,
|
||||||
|
});
|
||||||
|
}
|
||||||
|
jsonFile(join(output, "index.json"), { study: dir, rounds: index });
|
||||||
|
console.log(`审阅材料：${output}`);
|
||||||
@@ -0,0 +1,325 @@
|
|||||||
|
import { execFileSync } from "node:child_process";
|
||||||
|
import {
|
||||||
|
appendFileSync,
|
||||||
|
existsSync,
|
||||||
|
mkdirSync,
|
||||||
|
readFileSync,
|
||||||
|
writeFileSync,
|
||||||
|
} from "node:fs";
|
||||||
|
import { loadavg, freemem, totalmem } from "node:os";
|
||||||
|
import { join, resolve } from "node:path";
|
||||||
|
import { fullAccounts, FULL_STOP } from "../lib/full.ts";
|
||||||
|
import { jsonFile } from "../lib/forward.ts";
|
||||||
|
import {
|
||||||
|
sameContainerThroughResult,
|
||||||
|
type ContainerSnapshot,
|
||||||
|
} from "../lib/forward-evidence.ts";
|
||||||
|
import { ofKind, readLedger } from "../lib/ledger.ts";
|
||||||
|
import { launch, sleep, waitForLedger, writeConfig } from "../lib/study.ts";
|
||||||
|
import { watchTimeline } from "../lib/timeline.ts";
|
||||||
|
|
||||||
|
// 任务书 17 第 3.1 节：从头真实跑一份完整研究，到余额用完停在"导出还没写"。三份同时跑，由父会话各起一个进程；
|
||||||
|
// 容器名带研究编号，这里只认本研究的，不和别的研究互相核对。
|
||||||
|
// node eval/full/run.ts <研究编号> <全新绝对输出目录> [--kill-after-first-ablation-start]
|
||||||
|
// 服务一时调不通等中断后的续跑：npm run eval:study -- --resume --dir <输出目录>/study（照任务书 16 的做法）。
|
||||||
|
|
||||||
|
const [name, output, ...extra] = process.argv.slice(2);
|
||||||
|
const kill =
|
||||||
|
extra.length === 1 && extra[0] === "--kill-after-first-ablation-start";
|
||||||
|
if (
|
||||||
|
!name ||
|
||||||
|
!output ||
|
||||||
|
extra.length > 1 ||
|
||||||
|
(extra.length === 1 && !kill) ||
|
||||||
|
output !== resolve(output)
|
||||||
|
) {
|
||||||
|
throw new Error(
|
||||||
|
"用法：node eval/full/run.ts <研究编号> <全新绝对输出目录> [--kill-after-first-ablation-start]",
|
||||||
|
);
|
||||||
|
}
|
||||||
|
const out = output;
|
||||||
|
if (existsSync(out)) throw new Error(`输出目录已存在，不覆盖现场：${out}`);
|
||||||
|
mkdirSync(out, { recursive: true });
|
||||||
|
const studyDir = join(out, "study");
|
||||||
|
const config = writeConfig(
|
||||||
|
"eval/round-zero/research.json",
|
||||||
|
join(out, "research.json"),
|
||||||
|
name,
|
||||||
|
null,
|
||||||
|
);
|
||||||
|
|
||||||
|
const daemonTime = () =>
|
||||||
|
execFileSync("docker", ["info", "--format", "{{.SystemTime}}"], {
|
||||||
|
encoding: "utf8",
|
||||||
|
}).trim();
|
||||||
|
const inspect = (
|
||||||
|
container: string,
|
||||||
|
): ContainerSnapshot & {
|
||||||
|
Name: string;
|
||||||
|
Created: string;
|
||||||
|
State: {
|
||||||
|
Running: boolean;
|
||||||
|
StartedAt: string;
|
||||||
|
FinishedAt: string;
|
||||||
|
Status: string;
|
||||||
|
};
|
||||||
|
} =>
|
||||||
|
JSON.parse(
|
||||||
|
execFileSync("docker", ["inspect", container], { encoding: "utf8" }),
|
||||||
|
)[0];
|
||||||
|
|
||||||
|
/** 事件点的重快照（进程列表、容器列表），照任务书 16 的 machine.jsonl 做法。 */
|
||||||
|
function observe(label: string) {
|
||||||
|
appendFileSync(
|
||||||
|
join(out, "machine.jsonl"),
|
||||||
|
`${JSON.stringify({
|
||||||
|
at: new Date().toISOString(),
|
||||||
|
label,
|
||||||
|
processes: execFileSync("ps", ["-eo", "pid,ppid,lstart,args"], {
|
||||||
|
encoding: "utf8",
|
||||||
|
}),
|
||||||
|
containers: execFileSync("docker", ["ps", "--no-trunc"], {
|
||||||
|
encoding: "utf8",
|
||||||
|
}),
|
||||||
|
})}\n`,
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
/** 运行期间的轻量负载采样，几秒一条（任务书 17 第 3.1 节：和每次运行的 CPU/墙钟比对照，只列事实）。 */
|
||||||
|
function sample() {
|
||||||
|
appendFileSync(
|
||||||
|
join(out, "machine.jsonl"),
|
||||||
|
`${JSON.stringify({ at: new Date().toISOString(), label: "sample", load: loadavg(), memory: { total: totalmem(), free: freemem() } })}\n`,
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
function start(path: string | null) {
|
||||||
|
observe(path === null ? "resume" : "start");
|
||||||
|
const study = launch(path, studyDir, {});
|
||||||
|
appendFileSync(
|
||||||
|
join(out, "launches.jsonl"),
|
||||||
|
`${JSON.stringify({ event: "start", pid: study.child.pid, at: new Date().toISOString(), config: path })}\n`,
|
||||||
|
);
|
||||||
|
const interrupt = () => study.child.kill("SIGINT");
|
||||||
|
process.on("SIGINT", interrupt);
|
||||||
|
study.done.then((end) => {
|
||||||
|
process.off("SIGINT", interrupt);
|
||||||
|
appendFileSync(
|
||||||
|
join(out, "launches.jsonl"),
|
||||||
|
`${JSON.stringify({ event: "end", pid: study.child.pid, at: new Date().toISOString(), ...end })}\n`,
|
||||||
|
);
|
||||||
|
});
|
||||||
|
return study;
|
||||||
|
}
|
||||||
|
|
||||||
|
const sampler = setInterval(sample, 5000);
|
||||||
|
sampler.unref();
|
||||||
|
let active = start(config);
|
||||||
|
let killEvidence: {
|
||||||
|
run: string;
|
||||||
|
cell: string;
|
||||||
|
round: number;
|
||||||
|
killSeq: number;
|
||||||
|
} | null = null;
|
||||||
|
try {
|
||||||
|
await waitForLedger(studyDir, active.child);
|
||||||
|
let watching = watchTimeline(
|
||||||
|
studyDir,
|
||||||
|
active.child,
|
||||||
|
join(out, "timeline.jsonl"),
|
||||||
|
);
|
||||||
|
if (kill) {
|
||||||
|
// 盯住账本：第一个用途为 ablation 的运行确认启动之后、结果落账之前杀控制进程（照 eval/forward/run.ts 的 kill 做法）。
|
||||||
|
let hit = false;
|
||||||
|
while (active.child.exitCode === null && active.child.signalCode === null) {
|
||||||
|
const records = readLedger(studyDir);
|
||||||
|
const plan = ofKind(records, "runPlanned").find(
|
||||||
|
(r) => r.content.purpose === "ablation",
|
||||||
|
);
|
||||||
|
if (plan !== undefined) {
|
||||||
|
const run = plan.content.run;
|
||||||
|
if (ofKind(records, "runResult").some((r) => r.content.run === run)) {
|
||||||
|
throw new Error(
|
||||||
|
"第一个消融运行已经有结果了，未打中；保留现场，不改杀别的格",
|
||||||
|
);
|
||||||
|
}
|
||||||
|
const started = ofKind(records, "runStarted").find(
|
||||||
|
(r) => r.content.run === run,
|
||||||
|
);
|
||||||
|
if (started !== undefined) {
|
||||||
|
// 容器名是正式入口写在结构文档里的约定（inforsi-<研究编号>-<运行编号>），不 import flow 内部文件。
|
||||||
|
const container = `inforsi-${name}-${run}`;
|
||||||
|
// 从杀前划定历史事件区间，覆盖恢复启动到目标结果落账；不把 spawn 当成恢复已执行。
|
||||||
|
const lifecycleSince = daemonTime();
|
||||||
|
const before = inspect(container);
|
||||||
|
jsonFile(join(out, "container-before.json"), before);
|
||||||
|
if (!before.State.Running)
|
||||||
|
throw new Error("第一个消融容器已不在运行，未打中");
|
||||||
|
jsonFile(join(out, "kill.json"), {
|
||||||
|
run,
|
||||||
|
cell: plan.content.object,
|
||||||
|
round: plan.content.round,
|
||||||
|
planSeq: plan.seq,
|
||||||
|
startSeq: started.seq,
|
||||||
|
beforeKillSeq: records.at(-1)!.seq,
|
||||||
|
pid: active.child.pid,
|
||||||
|
at: new Date().toISOString(),
|
||||||
|
container,
|
||||||
|
});
|
||||||
|
if (!active.child.kill("SIGKILL")) throw new Error("未发出 SIGKILL");
|
||||||
|
const killed = await active.done;
|
||||||
|
await watching;
|
||||||
|
if (killed.signal !== "SIGKILL")
|
||||||
|
throw new Error("控制进程不是以 SIGKILL 结束");
|
||||||
|
const atKill = readLedger(studyDir);
|
||||||
|
jsonFile(join(out, "ledger-at-kill.json"), atKill);
|
||||||
|
if (ofKind(atKill, "runResult").some((r) => r.content.run === run)) {
|
||||||
|
throw new Error("杀后核对发现目标运行已经写结果，未打中；保留现场");
|
||||||
|
}
|
||||||
|
jsonFile(join(out, "container-after-kill.json"), inspect(container));
|
||||||
|
observe("after-kill");
|
||||||
|
jsonFile(
|
||||||
|
join(out, "container-before-resume.json"),
|
||||||
|
inspect(container),
|
||||||
|
);
|
||||||
|
killEvidence = {
|
||||||
|
run,
|
||||||
|
cell: plan.content.object!,
|
||||||
|
round: plan.content.round,
|
||||||
|
killSeq: atKill.at(-1)!.seq,
|
||||||
|
};
|
||||||
|
active = start(null);
|
||||||
|
watching = watchTimeline(
|
||||||
|
studyDir,
|
||||||
|
active.child,
|
||||||
|
join(out, "timeline.jsonl"),
|
||||||
|
);
|
||||||
|
let targetResult;
|
||||||
|
while (true) {
|
||||||
|
targetResult = ofKind(readLedger(studyDir), "runResult").find(
|
||||||
|
(r) => r.content.run === run,
|
||||||
|
);
|
||||||
|
if (
|
||||||
|
targetResult !== undefined ||
|
||||||
|
active.child.exitCode !== null ||
|
||||||
|
active.child.signalCode !== null
|
||||||
|
)
|
||||||
|
break;
|
||||||
|
await sleep(100);
|
||||||
|
}
|
||||||
|
const lifecycleUntil = daemonTime();
|
||||||
|
// Docker CLI 按时间回读，不依赖事件订阅的启动时序；原文全存，stats 核对缓存上限和目标 die。
|
||||||
|
writeFileSync(
|
||||||
|
join(out, "container-events.jsonl"),
|
||||||
|
execFileSync(
|
||||||
|
"docker",
|
||||||
|
[
|
||||||
|
"events",
|
||||||
|
"--since",
|
||||||
|
lifecycleSince,
|
||||||
|
"--until",
|
||||||
|
lifecycleUntil,
|
||||||
|
"--format",
|
||||||
|
"{{json .}}",
|
||||||
|
],
|
||||||
|
{ encoding: "utf8" },
|
||||||
|
),
|
||||||
|
);
|
||||||
|
jsonFile(join(out, "container-events-window.json"), {
|
||||||
|
since: lifecycleSince,
|
||||||
|
until: lifecycleUntil,
|
||||||
|
run,
|
||||||
|
resultSeq: targetResult?.seq ?? null,
|
||||||
|
});
|
||||||
|
if (targetResult === undefined)
|
||||||
|
throw new Error(
|
||||||
|
"恢复进程没有交出目标结果，生命周期证据不完整；保留现场",
|
||||||
|
);
|
||||||
|
hit = true;
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
await sleep(100);
|
||||||
|
}
|
||||||
|
if (!hit)
|
||||||
|
throw new Error(
|
||||||
|
`没有打中第一个消融运行：${JSON.stringify(await active.done)}`,
|
||||||
|
);
|
||||||
|
}
|
||||||
|
await watching;
|
||||||
|
const end = await active.done;
|
||||||
|
jsonFile(join(out, "outcome.json"), end);
|
||||||
|
observe("end");
|
||||||
|
const records = readLedger(studyDir);
|
||||||
|
const accounts = fullAccounts(records);
|
||||||
|
jsonFile(join(out, "accounts.json"), accounts);
|
||||||
|
|
||||||
|
// 杀进程的三条检查（任务书 17 第 3.1 节）：①恢复按容器名接着等同一个容器、没有重跑；②只扣一次；③结果照常写入、结算和后续步骤走完。
|
||||||
|
if (killEvidence !== null) {
|
||||||
|
const evidence = JSON.parse(
|
||||||
|
readFileSync(join(out, "kill.json"), "utf8"),
|
||||||
|
) as { run: string; cell: string; round: number; container: string };
|
||||||
|
const snapshots = (["before", "after-kill", "before-resume"] as const).map(
|
||||||
|
(part) =>
|
||||||
|
JSON.parse(
|
||||||
|
readFileSync(join(out, `container-${part}.json`), "utf8"),
|
||||||
|
) as ContainerSnapshot,
|
||||||
|
);
|
||||||
|
const lifecycle = readFileSync(join(out, "container-events.jsonl"), "utf8")
|
||||||
|
.split("\n")
|
||||||
|
.filter(Boolean)
|
||||||
|
.map((line) => JSON.parse(line));
|
||||||
|
const lifecycleChecks = sameContainerThroughResult(
|
||||||
|
snapshots,
|
||||||
|
lifecycle,
|
||||||
|
evidence.container,
|
||||||
|
);
|
||||||
|
const target = accounts.runs.find(
|
||||||
|
(r) => r.plan.content.run === evidence.run,
|
||||||
|
)!;
|
||||||
|
const cellPlans = accounts.runs.filter(
|
||||||
|
(r) =>
|
||||||
|
r.plan.content.purpose === "ablation" &&
|
||||||
|
r.plan.content.object === evidence.cell,
|
||||||
|
);
|
||||||
|
const report = {
|
||||||
|
evidence,
|
||||||
|
killSeq: killEvidence.killSeq,
|
||||||
|
snapshotMeaning: "杀前、杀后、恢复启动前；没有把快照当成恢复完成的证据",
|
||||||
|
lifecycleChecks,
|
||||||
|
checks: {
|
||||||
|
sameContainerNoRerun:
|
||||||
|
Object.values(lifecycleChecks).every(Boolean) &&
|
||||||
|
cellPlans.length === 1 &&
|
||||||
|
target.starts.length === 1,
|
||||||
|
chargedOnce: target.starts.length === 1,
|
||||||
|
resultSettledAndFinished:
|
||||||
|
target.results.length === 1 &&
|
||||||
|
target.results[0]!.seq > killEvidence.killSeq &&
|
||||||
|
ofKind(records, "settlement").some(
|
||||||
|
(s) => s.content.round === evidence.round,
|
||||||
|
) &&
|
||||||
|
accounts.halt !== null &&
|
||||||
|
accounts.halt.reason === FULL_STOP,
|
||||||
|
},
|
||||||
|
};
|
||||||
|
jsonFile(join(out, "kill-report.json"), report);
|
||||||
|
}
|
||||||
|
|
||||||
|
if (
|
||||||
|
end.code !== 0 ||
|
||||||
|
!accounts.checks.haltedAtExport ||
|
||||||
|
!accounts.checks.chargedEqualsStarts
|
||||||
|
) {
|
||||||
|
throw new Error(
|
||||||
|
`没有走到"导出还没写"或账目不平，保留现场；查明原因后用 npm run eval:study -- --resume --dir ${studyDir} 续跑：${JSON.stringify(end)}`,
|
||||||
|
);
|
||||||
|
}
|
||||||
|
console.log(`研究走完：${studyDir}。仍需 export、stats 和独立质量审阅。`);
|
||||||
|
} finally {
|
||||||
|
clearInterval(sampler);
|
||||||
|
if (active.child.exitCode === null && active.child.signalCode === null) {
|
||||||
|
active.child.kill("SIGINT");
|
||||||
|
await active.done;
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,180 @@
|
|||||||
|
import { existsSync, mkdirSync, readFileSync, writeFileSync } from "node:fs";
|
||||||
|
import { join, resolve } from "node:path";
|
||||||
|
import type { TaskMaterial } from "../../src/materials/index.ts";
|
||||||
|
import { cpuRatio, CPU_RATIO_FLAG, fullAccounts } from "../lib/full.ts";
|
||||||
|
import { frozenConfig, jsonFile } from "../lib/forward.ts";
|
||||||
|
import { ofKind, readLedger } from "../lib/ledger.ts";
|
||||||
|
import { lightCheckUsage } from "../lib/light-checks.ts";
|
||||||
|
import { readTimeline } from "../lib/timeline.ts";
|
||||||
|
|
||||||
|
// 任务书 17 第 3.2 节第 6、7 样的可计算部分：逐轮走向 + 程序账目 + CPU/墙钟比。只算事实、不作质量判断
|
||||||
|
// （解释质量、核验判得对不对、收尾与接力由父会话派全新上下文的子 Agent 读 eval/full/export.ts 的产物）。
|
||||||
|
// 这次三份研究同时跑，没有"启动区间互不重叠"这条（任务书 17 第 3.1 节）；也不设总体 acceptance。
|
||||||
|
// node eval/full/stats.ts <全新绝对统计输出目录> <研究输出目录>...
|
||||||
|
const [output, ...dirs] = process.argv.slice(2);
|
||||||
|
if (!output || dirs.length === 0 || output !== resolve(output)) {
|
||||||
|
throw new Error(
|
||||||
|
"用法：node eval/full/stats.ts <全新绝对统计输出目录> <研究输出目录>...",
|
||||||
|
);
|
||||||
|
}
|
||||||
|
mkdirSync(output);
|
||||||
|
|
||||||
|
const fmt = (v: number | null) => (v === null ? "—" : v.toFixed(6));
|
||||||
|
|
||||||
|
const summaries = dirs.map((dir) => {
|
||||||
|
const studyDir = join(dir, "study");
|
||||||
|
const records = readLedger(studyDir);
|
||||||
|
const config = frozenConfig(records);
|
||||||
|
const accounts = fullAccounts(records);
|
||||||
|
|
||||||
|
// 轻量检查：每次实现/修复/消融实现调用用了几次、有没有超过配置上限（任务书 17 第 3.2 节第 7 样）。
|
||||||
|
const lightChecks = ofKind(records, "callStarted")
|
||||||
|
.filter((c) =>
|
||||||
|
["implementation", "repair", "ablationImplementation"].includes(
|
||||||
|
(c.content.materials as TaskMaterial).kind,
|
||||||
|
),
|
||||||
|
)
|
||||||
|
.map((c) => {
|
||||||
|
const sessions = join(studyDir, "calls", c.content.call, "sessions");
|
||||||
|
return {
|
||||||
|
call: c.content.call,
|
||||||
|
seq: c.seq,
|
||||||
|
...(existsSync(sessions)
|
||||||
|
? lightCheckUsage(sessions, config.lightChecks.maxCount)
|
||||||
|
: {
|
||||||
|
maxCount: config.lightChecks.maxCount,
|
||||||
|
pass: false,
|
||||||
|
note: "会话目录不存在",
|
||||||
|
}),
|
||||||
|
};
|
||||||
|
});
|
||||||
|
|
||||||
|
// 每次正式运行的 CPU 时间/墙钟时间之比；比值明显低于 1 的列出，并和 machine.jsonl 同时段负载放在一起（只列事实）。
|
||||||
|
const timeline = existsSync(join(dir, "timeline.jsonl"))
|
||||||
|
? readTimeline(join(dir, "timeline.jsonl"))
|
||||||
|
: [];
|
||||||
|
const atOf = (seq: number) => timeline.find((t) => t.seq === seq)?.at ?? null;
|
||||||
|
const machine = existsSync(join(dir, "machine.jsonl"))
|
||||||
|
? readFileSync(join(dir, "machine.jsonl"), "utf8")
|
||||||
|
.split("\n")
|
||||||
|
.filter(Boolean)
|
||||||
|
.map(
|
||||||
|
(line) =>
|
||||||
|
JSON.parse(line) as { at: string; label: string; load?: number[] },
|
||||||
|
)
|
||||||
|
: [];
|
||||||
|
const ratios = accounts.runs.map((r) => {
|
||||||
|
const result = r.results[0] ?? null;
|
||||||
|
const ratio = result === null ? null : cpuRatio(result.content);
|
||||||
|
const start = r.starts[0] === undefined ? null : atOf(r.starts[0].seq);
|
||||||
|
const end = result === null ? null : atOf(result.seq);
|
||||||
|
const samples =
|
||||||
|
start === null || end === null
|
||||||
|
? []
|
||||||
|
: machine
|
||||||
|
.filter(
|
||||||
|
(m) =>
|
||||||
|
m.label === "sample" &&
|
||||||
|
Date.parse(m.at) >= start &&
|
||||||
|
Date.parse(m.at) <= end,
|
||||||
|
)
|
||||||
|
.map((m) => ({ at: m.at, load: m.load }));
|
||||||
|
return {
|
||||||
|
run: r.plan.content.run,
|
||||||
|
purpose: r.plan.content.purpose,
|
||||||
|
round: r.plan.content.round,
|
||||||
|
object: r.plan.content.object,
|
||||||
|
ratio,
|
||||||
|
flagged: ratio !== null && ratio < CPU_RATIO_FLAG,
|
||||||
|
start: start === null ? null : new Date(start).toISOString(),
|
||||||
|
end: end === null ? null : new Date(end).toISOString(),
|
||||||
|
loadSamples: samples,
|
||||||
|
note: `比值低于 ${CPU_RATIO_FLAG} 列出并附同时段负载；只列事实，不作判断`,
|
||||||
|
};
|
||||||
|
});
|
||||||
|
|
||||||
|
// 有 kill-report.json 的研究，把三条检查的结果抄进来（任务书 17 第 3.1 节）。
|
||||||
|
const kill = existsSync(join(dir, "kill-report.json"))
|
||||||
|
? (JSON.parse(readFileSync(join(dir, "kill-report.json"), "utf8")) as {
|
||||||
|
evidence: unknown;
|
||||||
|
checks: Record<string, boolean>;
|
||||||
|
})
|
||||||
|
: null;
|
||||||
|
|
||||||
|
const pass =
|
||||||
|
Object.values(accounts.checks).every(Boolean) &&
|
||||||
|
lightChecks.every((c) => c.pass) &&
|
||||||
|
(kill === null || Object.values(kill.checks).every(Boolean));
|
||||||
|
return {
|
||||||
|
dir,
|
||||||
|
study: config.studyId,
|
||||||
|
accounts,
|
||||||
|
lightChecks,
|
||||||
|
ratios,
|
||||||
|
kill,
|
||||||
|
pass,
|
||||||
|
};
|
||||||
|
});
|
||||||
|
|
||||||
|
jsonFile(join(output, "stats.json"), summaries);
|
||||||
|
|
||||||
|
const text = summaries.map((s) => {
|
||||||
|
const a = s.accounts;
|
||||||
|
const lines = [
|
||||||
|
`研究 ${s.study}（${s.dir}）`,
|
||||||
|
`确认启动：基线 ${a.confirmedStarts.baseline}、正向 ${a.confirmedStarts.forward}、消融 ${a.confirmedStarts.ablation}，共 ${a.confirmedStarts.total}；扣次 ${a.balance.charged}，余额 ${a.balance.balance}，占用 ${a.balance.reserved}；已知最好 ${fmt(a.knownBest)}`,
|
||||||
|
`停下：${a.halt === null ? "（没有停下事件）" : a.halt.reason}`,
|
||||||
|
`账目核对：${Object.entries(a.checks)
|
||||||
|
.map(([k, v]) => `${k}=${v ? "过" : "不过"}`)
|
||||||
|
.join("、")}`,
|
||||||
|
`轻量检查：${s.lightChecks.every((c) => c.pass) ? "全部在上限内" : "有超限或会话缺失，见 stats.json"}`,
|
||||||
|
`CPU/墙钟比低于 ${CPU_RATIO_FLAG} 的运行：${
|
||||||
|
s.ratios
|
||||||
|
.filter((r) => r.flagged)
|
||||||
|
.map((r) => `${r.run}=${r.ratio!.toFixed(3)}`)
|
||||||
|
.join("、") || "没有"
|
||||||
|
}`,
|
||||||
|
s.kill === null
|
||||||
|
? "杀进程：这份没做"
|
||||||
|
: `杀进程：${Object.entries(s.kill.checks)
|
||||||
|
.map(([k, v]) => `${k}=${v ? "过" : "不过"}`)
|
||||||
|
.join("、")}`,
|
||||||
|
"",
|
||||||
|
];
|
||||||
|
for (const r of a.rounds) {
|
||||||
|
lines.push(
|
||||||
|
`第 ${r.round} 轮（基线 ${r.baseline}）`,
|
||||||
|
...r.candidates.map(
|
||||||
|
(c) =>
|
||||||
|
` 候选 ${c.candidate}：实测 ${fmt(c.score)}，合并 ${c.merged === null ? "—" : `${fmt(c.merged.q1)}/${fmt(c.merged.q5)}/${fmt(c.merged.q9)}`}，落点 ${c.band ?? "—"}，作者自评 ${c.authorEstimates.map((u) => `${u.author} ${fmt(u.range.q1)}/${fmt(u.range.q5)}/${fmt(u.range.q9)}`).join("、") || "—"}${c.notRunEvents.length > 0 ? "，不跑：" + c.notRunEvents[0]!.reason : ""}`,
|
||||||
|
),
|
||||||
|
` 采纳：${r.adopted ?? "没有"}${r.adoptedScore === null ? "" : `，分数 ${fmt(r.adoptedScore)}，离已知最好 ${fmt(r.knownBestGap)}`}`,
|
||||||
|
);
|
||||||
|
if (r.hasAblation) {
|
||||||
|
lines.push(
|
||||||
|
` 消融：选格 ${r.cellsDecision!.output.chosen.join("、") || "（一格都没选）"}；配合关系 ${r.label === null ? "—" : r.label.label}；押的因子 ${r.bet === null ? "—" : `${r.bet.decision === "bet" ? r.bet.factor : `不押（${r.bet.decision}）`}，预测收益 ${fmt(r.bet.predictedGain)}`}`,
|
||||||
|
...r.cells.map(
|
||||||
|
(c) =>
|
||||||
|
` ${c.cell}：核验 ${c.verificationCount} 次、重做 ${c.redoCount} 次${c.cannotRun === null ? "" : "，跑不起来：" + c.cannotRun}${c.notRunEvents.length > 0 ? "，不跑：" + c.notRunEvents[0]!.reason : ""}，跑没跑 ${c.ran ? "跑了" : "没跑"}（扣次 ${c.charged}），实测 ${fmt(c.measured)}，撑几成 ${c.support === null ? "—" : c.support.toFixed(3)}，合并 ${c.merged === null ? "—" : `${fmt(c.merged.q1)}/${fmt(c.merged.q5)}/${fmt(c.merged.q9)}`}，结局 ${c.outcome ?? "—"}`,
|
||||||
|
),
|
||||||
|
...r
|
||||||
|
.cellsDecision!.output.cells.filter((c) => !c.tested)
|
||||||
|
.map(
|
||||||
|
(c) =>
|
||||||
|
` ${c.factor} 没测：${c.reason}（含它的最好组合净值 ${c.bestCombo === null ? "没有可选组合" : c.bestCombo.net.toExponential(3)}）`,
|
||||||
|
),
|
||||||
|
);
|
||||||
|
} else lines.push(" 消融：这一轮没有");
|
||||||
|
lines.push(
|
||||||
|
` 解释关闭早于首格启动：${r.explanationBeforeCells === null ? "无消融" : r.explanationBeforeCells ? "是" : "否"}；解封在收尾关闭之后且覆盖选中候选：${r.unseal.pass ? "是" : "否"}`,
|
||||||
|
);
|
||||||
|
lines.push("");
|
||||||
|
}
|
||||||
|
return lines.join("\n");
|
||||||
|
});
|
||||||
|
writeFileSync(join(output, "stats.txt"), `${text.join("\n")}\n`);
|
||||||
|
|
||||||
|
console.log(
|
||||||
|
`统计：${output}；所给 ${summaries.length} 份研究的逐研究账目检查${summaries.every((s) => s.pass) ? "通过" : "未通过"}。不设总体 acceptance（三份同时跑，无互不重叠条款）；研究质量判断仍待独立审阅。`,
|
||||||
|
);
|
||||||
|
if (!summaries.every((s) => s.pass)) process.exitCode = 1;
|
||||||
@@ -0,0 +1,376 @@
|
|||||||
|
import type { LedgerRecord } from "../../src/ledger/index.ts";
|
||||||
|
import {
|
||||||
|
Board,
|
||||||
|
CELL_NOT_RUN,
|
||||||
|
cellId,
|
||||||
|
FORWARD_NOT_RUN,
|
||||||
|
ORIGINAL_BASELINE,
|
||||||
|
validScore,
|
||||||
|
type StudyRecord,
|
||||||
|
} from "../../src/materials/index.ts";
|
||||||
|
import { improvement, runBalance } from "../../src/rules/index.ts";
|
||||||
|
import { ablationOutcome } from "../../src/submissions/index.ts";
|
||||||
|
import { checkTaskFacts } from "../../src/tasks/index.ts";
|
||||||
|
import { frozenConfig, scoreBand } from "./forward.ts";
|
||||||
|
import { ofKind } from "./ledger.ts";
|
||||||
|
|
||||||
|
// 完整研究（任务书 17 第三块）的账目核对：只从账本记录重算事实，不作质量判断。
|
||||||
|
// 质量判断（解释好不好、核验判得对不对、收尾写得怎么样）由父会话派全新上下文的子 Agent 做。
|
||||||
|
|
||||||
|
/** 研究走完一轮接一轮、余额用完后的停点（任务书 17 第 2.1 节）。 */
|
||||||
|
export const FULL_STOP = "导出还没写";
|
||||||
|
|
||||||
|
/** CPU 时间 / 墙钟时间低于这个比值就列出来和机器负载放在一起（任务书 17 第 3.1 节"明显低于 1"；只列事实）。 */
|
||||||
|
export const CPU_RATIO_FLAG = 0.9;
|
||||||
|
|
||||||
|
/** 一份完整研究的账目：每轮的走向 + 全研究的程序账目核对（任务书 17 第 3.2 节第 6、7 样的可计算部分）。 */
|
||||||
|
export function fullAccounts(records: LedgerRecord[]) {
|
||||||
|
const board = new Board(records);
|
||||||
|
const config = frozenConfig(records);
|
||||||
|
// SAFETY: 账本 config 记录的内容形状由 materials 的 StudyRecord 定义（docs/结构.md 第八部分），ledger 里只存 Json。
|
||||||
|
const studyRecord = ofKind(records, "config")[0]!
|
||||||
|
.content as unknown as StudyRecord;
|
||||||
|
const facts = checkTaskFacts(studyRecord.task);
|
||||||
|
const events = ofKind(records, "event").map((r) => ({
|
||||||
|
...r.content,
|
||||||
|
seq: r.seq,
|
||||||
|
}));
|
||||||
|
const started = ofKind(records, "runStarted");
|
||||||
|
const runs = ofKind(records, "runPlanned").map((p) => ({
|
||||||
|
plan: p,
|
||||||
|
starts: started.filter((s) => s.content.run === p.content.run),
|
||||||
|
results: ofKind(records, "runResult").filter(
|
||||||
|
(r) => r.content.run === p.content.run,
|
||||||
|
),
|
||||||
|
}));
|
||||||
|
const balance = runBalance(config.runs, board.runStates());
|
||||||
|
|
||||||
|
// 阶段开关成对：每个开恰有一个比它晚的关；没有没开的关。
|
||||||
|
const opens = ofKind(records, "stageOpen");
|
||||||
|
const closes = ofKind(records, "stageClose");
|
||||||
|
const stages = opens.map((r) => {
|
||||||
|
const closing = closes.filter(
|
||||||
|
(c) => c.content.stageId === r.content.stageId,
|
||||||
|
);
|
||||||
|
return {
|
||||||
|
...r.content,
|
||||||
|
openSeq: r.seq,
|
||||||
|
closeSeq: closing[0]?.seq ?? null,
|
||||||
|
pass: closing.length === 1 && closing[0]!.seq > r.seq,
|
||||||
|
};
|
||||||
|
});
|
||||||
|
const orphanCloses = closes.filter(
|
||||||
|
(c) => !opens.some((o) => o.content.stageId === c.content.stageId),
|
||||||
|
);
|
||||||
|
const closeSeqOf = (stageId: string) =>
|
||||||
|
stages.find((s) => s.stageId === stageId)?.closeSeq ?? null;
|
||||||
|
|
||||||
|
const roundIds = [
|
||||||
|
...new Set([
|
||||||
|
...opens.map((o) => o.content.round),
|
||||||
|
...ofKind(records, "decision").map((d) => d.content.round),
|
||||||
|
...ofKind(records, "settlement").map((s) => s.content.round),
|
||||||
|
]),
|
||||||
|
]
|
||||||
|
.filter((r) => r >= 1)
|
||||||
|
.sort((a, b) => a - b);
|
||||||
|
|
||||||
|
const rounds = roundIds.map((round) => {
|
||||||
|
const batch = board.decision("batch", round);
|
||||||
|
const chosen = batch?.output.chosen ?? [];
|
||||||
|
// 每个入选候选一行（任务书 17 第 3.2 节第 6 样）：路线代号不自动猜，留给人工填。
|
||||||
|
const candidates = chosen.map((candidate) => {
|
||||||
|
const run = board.runOf(candidate);
|
||||||
|
const score = validScore(run?.result?.judged ?? null);
|
||||||
|
const merged =
|
||||||
|
batch!.output.candidates.find((c) => c.id === candidate)?.merged ??
|
||||||
|
null;
|
||||||
|
return {
|
||||||
|
candidate,
|
||||||
|
score,
|
||||||
|
merged,
|
||||||
|
authorEstimates: board
|
||||||
|
.unsealed()
|
||||||
|
.filter((u) => u.candidate === candidate),
|
||||||
|
band: merged === null ? null : scoreBand(score, merged),
|
||||||
|
started: run !== undefined,
|
||||||
|
notRunEvents: events.filter(
|
||||||
|
(e) => e.event === FORWARD_NOT_RUN && e.objects.includes(candidate),
|
||||||
|
),
|
||||||
|
route: "路线代号由父会话审阅时人工填，不自动按关键词贴",
|
||||||
|
};
|
||||||
|
});
|
||||||
|
const adopt = board.decision("adopt", round) ?? null;
|
||||||
|
const adopted = adopt?.output.adopted ?? null;
|
||||||
|
const adoptedScore =
|
||||||
|
adopted === null
|
||||||
|
? null
|
||||||
|
: validScore(board.runOf(adopted)?.result?.judged ?? null);
|
||||||
|
const bestSoFar =
|
||||||
|
board.decision("batch", round) === undefined ? null : null;
|
||||||
|
void bestSoFar;
|
||||||
|
const knownBestGap =
|
||||||
|
adoptedScore === null || facts.knownBest === null
|
||||||
|
? null
|
||||||
|
: improvement(
|
||||||
|
adoptedScore,
|
||||||
|
facts.knownBest,
|
||||||
|
facts.finalMetric.direction,
|
||||||
|
);
|
||||||
|
|
||||||
|
// 消融（任务书 17 第 3.2 节第 4 样的可计算部分和第 7 样的逐格账目）。
|
||||||
|
const cellsDecision = board.decision("cells", round) ?? null;
|
||||||
|
const settlement = board.settlement(round) ?? null;
|
||||||
|
const bet = board.decision("bet", round) ?? null;
|
||||||
|
const cells = (cellsDecision?.output.chosen ?? []).map((key) => {
|
||||||
|
const cell = cellId(adopted!, key);
|
||||||
|
// published 只收已关阶段的最后一版；研究走完了，这些阶段都已关闭。
|
||||||
|
const impls = board
|
||||||
|
.published("ablationImplementation")
|
||||||
|
.filter((s) => s.target === cell);
|
||||||
|
const lastImpl = impls.at(-1) ?? null;
|
||||||
|
const verifications = board
|
||||||
|
.published("ablationVerification")
|
||||||
|
.filter((s) => s.target === cell);
|
||||||
|
const lastVerification = verifications.at(-1) ?? null;
|
||||||
|
const lastOutcome =
|
||||||
|
lastVerification === null || lastImpl === null
|
||||||
|
? null
|
||||||
|
: ablationOutcome(
|
||||||
|
lastVerification.content,
|
||||||
|
key,
|
||||||
|
lastImpl.content.alsoOff,
|
||||||
|
);
|
||||||
|
const cellRuns = runs.filter(
|
||||||
|
(r) =>
|
||||||
|
r.plan.content.purpose === "ablation" &&
|
||||||
|
r.plan.content.round === round &&
|
||||||
|
r.plan.content.object === cell,
|
||||||
|
);
|
||||||
|
const lastRun = cellRuns.at(-1) ?? null;
|
||||||
|
const settled =
|
||||||
|
settlement?.output.cells.find((c) => c.factor === key) ?? null;
|
||||||
|
const settleInput =
|
||||||
|
settlement?.input.cells.find((c) => c.factor === key)?.outcome ?? null;
|
||||||
|
const notRun = events.filter(
|
||||||
|
(e) => e.event === CELL_NOT_RUN && e.objects.includes(cell),
|
||||||
|
);
|
||||||
|
const cannotRun = lastImpl === null ? null : lastImpl.content.cannotRun;
|
||||||
|
return {
|
||||||
|
cell,
|
||||||
|
factor: key,
|
||||||
|
implementationCount: impls.length,
|
||||||
|
redoCount: Math.max(0, impls.length - 1),
|
||||||
|
verificationCount: verifications.length,
|
||||||
|
cannotRun,
|
||||||
|
notRunEvents: notRun,
|
||||||
|
lastVerificationPassed:
|
||||||
|
lastOutcome === null ? null : lastOutcome.passed,
|
||||||
|
charged: cellRuns.filter((r) => r.starts.length > 0).length,
|
||||||
|
ran:
|
||||||
|
lastRun !== null &&
|
||||||
|
lastRun.results.length === 1 &&
|
||||||
|
lastRun.results[0]!.content.process.ending !== "notStarted",
|
||||||
|
measured:
|
||||||
|
settleInput !== null &&
|
||||||
|
(settleInput.kind === "valid" || settleInput.kind === "joint")
|
||||||
|
? settleInput.score
|
||||||
|
: null,
|
||||||
|
alsoOff:
|
||||||
|
settleInput !== null && settleInput.kind === "joint"
|
||||||
|
? settleInput.off
|
||||||
|
: [],
|
||||||
|
outcome: settled?.outcome ?? null,
|
||||||
|
support: settled?.support ?? null,
|
||||||
|
merged: settled?.merged ?? null,
|
||||||
|
worldBits: settled?.worldBits ?? null,
|
||||||
|
surprise: settled?.surprise ?? null,
|
||||||
|
// 每个选中的格都已结束这一段：交了"跑不起来"，或核验不通过有"不跑"事件，或最近一次核验通过且跑出了结果。
|
||||||
|
accounted:
|
||||||
|
lastImpl !== null &&
|
||||||
|
(cannotRun !== null ||
|
||||||
|
notRun.length > 0 ||
|
||||||
|
(lastOutcome?.passed === true &&
|
||||||
|
lastRun !== null &&
|
||||||
|
lastRun.results.length === 1 &&
|
||||||
|
lastRun.results[0]!.content.process.ending !== "notStarted")),
|
||||||
|
};
|
||||||
|
});
|
||||||
|
|
||||||
|
// 解释阶段关闭早于本轮任何一格的确认启动（任务书 17 第 3.2 节第 7 样）。
|
||||||
|
const explanationClose = closeSeqOf(`r${round}-explanation`);
|
||||||
|
const cellStartSeqs = runs
|
||||||
|
.filter(
|
||||||
|
(r) =>
|
||||||
|
r.plan.content.purpose === "ablation" &&
|
||||||
|
r.plan.content.round === round,
|
||||||
|
)
|
||||||
|
.flatMap((r) => r.starts.map((s) => s.seq));
|
||||||
|
const firstCellStart =
|
||||||
|
cellStartSeqs.length === 0 ? null : Math.min(...cellStartSeqs);
|
||||||
|
const explanationBeforeCells =
|
||||||
|
cellsDecision === null
|
||||||
|
? null
|
||||||
|
: explanationClose !== null &&
|
||||||
|
(firstCellStart === null || explanationClose < firstCellStart);
|
||||||
|
|
||||||
|
// 轮末解封：本轮选中的每个候选都有解封记录，且写在收尾阶段关闭之后（任务书 17 第 3.2 节第 7 样）。
|
||||||
|
const wrapUpClose = closeSeqOf(`r${round}-wrapUp`);
|
||||||
|
const unseals = ofKind(records, "unseal").filter((u) =>
|
||||||
|
chosen.includes(u.content.candidate),
|
||||||
|
);
|
||||||
|
const unseal = {
|
||||||
|
wrapUpClose,
|
||||||
|
perCandidate: chosen.map((candidate) => {
|
||||||
|
const u = unseals.find((x) => x.content.candidate === candidate);
|
||||||
|
return {
|
||||||
|
candidate,
|
||||||
|
unsealSeq: u?.seq ?? null,
|
||||||
|
afterWrapUpClose:
|
||||||
|
u !== undefined && wrapUpClose !== null && u.seq > wrapUpClose,
|
||||||
|
};
|
||||||
|
}),
|
||||||
|
pass:
|
||||||
|
wrapUpClose !== null &&
|
||||||
|
chosen.length > 0 &&
|
||||||
|
chosen.every((candidate) => {
|
||||||
|
const u = unseals.find((x) => x.content.candidate === candidate);
|
||||||
|
return u !== undefined && u.seq > wrapUpClose;
|
||||||
|
}),
|
||||||
|
};
|
||||||
|
|
||||||
|
return {
|
||||||
|
round,
|
||||||
|
baseline: board.baselineOf(round),
|
||||||
|
candidates,
|
||||||
|
adopted,
|
||||||
|
adoptedScore,
|
||||||
|
knownBestGap,
|
||||||
|
hasAblation: cellsDecision !== null,
|
||||||
|
cellsDecision,
|
||||||
|
cells,
|
||||||
|
label: settlement?.output.label ?? null,
|
||||||
|
judgments: settlement?.output.judgments ?? null,
|
||||||
|
inseparable: settlement?.output.inseparable ?? null,
|
||||||
|
finding: settlement?.output.finding ?? null,
|
||||||
|
bet:
|
||||||
|
bet === null
|
||||||
|
? null
|
||||||
|
: {
|
||||||
|
decision: bet.output.decision,
|
||||||
|
factor: bet.output.factor,
|
||||||
|
predictedGain: bet.output.predictedGain,
|
||||||
|
offBetter: bet.output.offBetter,
|
||||||
|
},
|
||||||
|
settlementCount: board.settlements().filter((s) => s.round === round)
|
||||||
|
.length,
|
||||||
|
explanationBeforeCells,
|
||||||
|
unseal,
|
||||||
|
};
|
||||||
|
});
|
||||||
|
|
||||||
|
// 第 2 轮起的基线：从采纳决定和结算的"消融中发现"逐轮重算，和 Board 现算的对照；
|
||||||
|
// 再核对第 r 轮冻结的协议都写在当时的基线上（方法.md 4.3 ①）。
|
||||||
|
const chain: Array<{ round: number; baseline: string }> = [
|
||||||
|
{ round: 1, baseline: ORIGINAL_BASELINE },
|
||||||
|
];
|
||||||
|
for (const round of roundIds) {
|
||||||
|
const adopted = board.decision("adopt", round)?.output.adopted ?? null;
|
||||||
|
const found = board.settlement(round)?.output.finding.chosen ?? null;
|
||||||
|
const previous = chain.at(-1)!.baseline;
|
||||||
|
chain.push({
|
||||||
|
round: round + 1,
|
||||||
|
baseline:
|
||||||
|
found === null
|
||||||
|
? adopted === null
|
||||||
|
? previous
|
||||||
|
: adopted
|
||||||
|
: cellId(adopted!, found),
|
||||||
|
});
|
||||||
|
}
|
||||||
|
const baselineChain = {
|
||||||
|
chain,
|
||||||
|
boardAgrees: chain.every((c) => board.baselineOf(c.round) === c.baseline),
|
||||||
|
protocolsOnBaseline: board
|
||||||
|
.protocols()
|
||||||
|
.filter((p) => p.protocol.round >= 2)
|
||||||
|
.every(
|
||||||
|
(p) =>
|
||||||
|
p.protocol.baseline ===
|
||||||
|
chain.find((c) => c.round === p.protocol.round)?.baseline,
|
||||||
|
),
|
||||||
|
};
|
||||||
|
|
||||||
|
// "跑不起来"和"不跑"的格不扣次（任务书 17 第 3.2 节第 7 样）：这些格没有任何运行登记。
|
||||||
|
const silentCells = rounds.flatMap((r) =>
|
||||||
|
r.cells.filter((c) => c.cannotRun !== null || c.notRunEvents.length > 0),
|
||||||
|
);
|
||||||
|
const silentNotCharged = silentCells.every((c) => c.charged === 0);
|
||||||
|
|
||||||
|
const halt = events.filter((e) => e.event === "halted").at(-1) ?? null;
|
||||||
|
const checks = {
|
||||||
|
stagesPaired: stages.every((s) => s.pass) && orphanCloses.length === 0,
|
||||||
|
chargedEqualsStarts: balance.charged === started.length,
|
||||||
|
finalBalanceZero: balance.balance === 0,
|
||||||
|
reservedZero: balance.reserved === 0,
|
||||||
|
runsFinished: runs.length > 0 && runs.every((r) => r.results.length === 1),
|
||||||
|
explanationBeforeCells: rounds.every(
|
||||||
|
(r) => r.explanationBeforeCells !== false,
|
||||||
|
),
|
||||||
|
oneSettlementPerRound: rounds.every((r) => r.settlementCount <= 1),
|
||||||
|
unsealAfterWrapUp: rounds.every((r) => r.unseal.pass),
|
||||||
|
cellsAccounted: rounds.every((r) => r.cells.every((c) => c.accounted)),
|
||||||
|
silentCellsNotCharged: silentNotCharged,
|
||||||
|
baselineChain:
|
||||||
|
baselineChain.boardAgrees && baselineChain.protocolsOnBaseline,
|
||||||
|
haltedAtExport:
|
||||||
|
halt !== null &&
|
||||||
|
halt.reason === FULL_STOP &&
|
||||||
|
halt.seq === records.at(-1)!.seq,
|
||||||
|
};
|
||||||
|
return {
|
||||||
|
study: config.studyId,
|
||||||
|
initial: config.runs,
|
||||||
|
knownBest: facts.knownBest,
|
||||||
|
confirmedStarts: {
|
||||||
|
total: started.length,
|
||||||
|
baseline: runs.filter(
|
||||||
|
(r) => r.plan.content.purpose === "baseline" && r.starts.length > 0,
|
||||||
|
).length,
|
||||||
|
forward: runs.filter(
|
||||||
|
(r) => r.plan.content.purpose === "forward" && r.starts.length > 0,
|
||||||
|
).length,
|
||||||
|
ablation: runs.filter(
|
||||||
|
(r) => r.plan.content.purpose === "ablation" && r.starts.length > 0,
|
||||||
|
).length,
|
||||||
|
},
|
||||||
|
balance,
|
||||||
|
runs,
|
||||||
|
rounds,
|
||||||
|
baselineChain,
|
||||||
|
stages,
|
||||||
|
orphanCloses,
|
||||||
|
halt,
|
||||||
|
checks,
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
/** 一次正式运行的 CPU 时间 / 墙钟时间（任务书 17 第 3.1 节）；测不到或没启动为 null。 */
|
||||||
|
export function cpuRatio(result: {
|
||||||
|
process: {
|
||||||
|
ending: string;
|
||||||
|
cpuSeconds?: { status: string; value?: number };
|
||||||
|
wallSeconds?: { status: string; value?: number };
|
||||||
|
};
|
||||||
|
}): number | null {
|
||||||
|
const p = result.process;
|
||||||
|
if (p.ending === "notStarted") return null;
|
||||||
|
if (
|
||||||
|
p.cpuSeconds?.status !== "measured" ||
|
||||||
|
p.wallSeconds?.status !== "measured"
|
||||||
|
)
|
||||||
|
return null;
|
||||||
|
return p.wallSeconds.value === 0
|
||||||
|
? null
|
||||||
|
: p.cpuSeconds.value! / p.wallSeconds.value!;
|
||||||
|
}
|
||||||
+4
-1
@@ -30,7 +30,10 @@
|
|||||||
"eval:route:thinking": "node eval/route/thinking.ts",
|
"eval:route:thinking": "node eval/route/thinking.ts",
|
||||||
"eval:forward:run": "set -a && . /home/iomgaa/Projects/RivalBits/.env && set +a && node eval/forward/run.ts",
|
"eval:forward:run": "set -a && . /home/iomgaa/Projects/RivalBits/.env && set +a && node eval/forward/run.ts",
|
||||||
"eval:forward:export": "node eval/forward/export.ts",
|
"eval:forward:export": "node eval/forward/export.ts",
|
||||||
"eval:forward:stats": "node eval/forward/stats.ts"
|
"eval:forward:stats": "node eval/forward/stats.ts",
|
||||||
|
"eval:full:run": "set -a && . /home/iomgaa/Projects/RivalBits/.env && set +a && node eval/full/run.ts",
|
||||||
|
"eval:full:stats": "node eval/full/stats.ts",
|
||||||
|
"eval:full:export": "node eval/full/export.ts"
|
||||||
},
|
},
|
||||||
"dependencies": {
|
"dependencies": {
|
||||||
"@earendil-works/pi-ai": "0.85.1",
|
"@earendil-works/pi-ai": "0.85.1",
|
||||||
|
|||||||
@@ -0,0 +1,448 @@
|
|||||||
|
import assert from "node:assert/strict";
|
||||||
|
import { spawnSync } from "node:child_process";
|
||||||
|
import { mkdirSync, readFileSync, writeFileSync } from "node:fs";
|
||||||
|
import { join } from "node:path";
|
||||||
|
import { test, type TestContext } from "node:test";
|
||||||
|
import { cpuRatio, fullAccounts, FULL_STOP } from "../eval/lib/full.ts";
|
||||||
|
import { Ledger } from "../src/ledger/index.ts";
|
||||||
|
import {
|
||||||
|
cellId,
|
||||||
|
cellNotRunEvent,
|
||||||
|
type Settlement,
|
||||||
|
} from "../src/materials/index.ts";
|
||||||
|
import {
|
||||||
|
chooseBet,
|
||||||
|
runCellRule,
|
||||||
|
settleAblation,
|
||||||
|
type AblationModelInput,
|
||||||
|
type CellRuleInput,
|
||||||
|
type SettleInput,
|
||||||
|
} from "../src/rules/index.ts";
|
||||||
|
import { checkTaskFacts } from "../src/tasks/index.ts";
|
||||||
|
import { createRepository, saveVersion } from "../src/workspace/index.ts";
|
||||||
|
import { fullConfig } from "./config-helpers.ts";
|
||||||
|
import { definition, exited, tempDir } from "./ledger-helpers.ts";
|
||||||
|
import { batch, judged, protocolOf, r, Writer } from "./materials-helpers.ts";
|
||||||
|
import type { Submission } from "../src/vocab/index.ts";
|
||||||
|
|
||||||
|
// eval/full 的账目统计与审阅导出：在真实账本里写一份走完一轮（含消融）的研究，跑 stats 和 export，看产物。
|
||||||
|
// 选格、结算、押的因子的记录用 rules 的真实函数现算（与 flow 同一个入口），预期值按方法.md 6.6 手算对照。
|
||||||
|
|
||||||
|
const facts = checkTaskFacts(
|
||||||
|
JSON.parse(
|
||||||
|
readFileSync(
|
||||||
|
join(import.meta.dirname, "..", "tasks", "circle_packing", "task.json"),
|
||||||
|
"utf8",
|
||||||
|
),
|
||||||
|
),
|
||||||
|
);
|
||||||
|
|
||||||
|
const TWO_FACTORS: Submission<"proposal"> = {
|
||||||
|
factors: [
|
||||||
|
{
|
||||||
|
name: "多起点重启",
|
||||||
|
method: "做法一",
|
||||||
|
keySettings: "设定一",
|
||||||
|
continues: null,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "停滞扰动",
|
||||||
|
method: "做法二",
|
||||||
|
keySettings: "设定二",
|
||||||
|
continues: null,
|
||||||
|
},
|
||||||
|
],
|
||||||
|
safeguards: [],
|
||||||
|
metrics: { defined: [], reused: [] },
|
||||||
|
authorEstimate: r(1.0, 1.1, 1.2),
|
||||||
|
body: "提案正文",
|
||||||
|
};
|
||||||
|
|
||||||
|
const EXPLANATIONS = [
|
||||||
|
{
|
||||||
|
author: "a1",
|
||||||
|
cells: [
|
||||||
|
{ factor: "p1", range: r(1.0, 1.1, 1.2) },
|
||||||
|
{ factor: "p2", range: r(1.15, 1.2, 1.24) },
|
||||||
|
],
|
||||||
|
},
|
||||||
|
{
|
||||||
|
author: "a2",
|
||||||
|
cells: [
|
||||||
|
{ factor: "p1", range: r(0.9, 1.05, 1.2) },
|
||||||
|
{ factor: "p2", range: r(1.1, 1.18, 1.25) },
|
||||||
|
],
|
||||||
|
},
|
||||||
|
];
|
||||||
|
|
||||||
|
const modelInput = (): AblationModelInput => ({
|
||||||
|
baseline: 0.96,
|
||||||
|
full: 1.25,
|
||||||
|
direction: "maximize",
|
||||||
|
knownBest: facts.knownBest,
|
||||||
|
epsilon: facts.epsilon,
|
||||||
|
lambda: 0.1,
|
||||||
|
factors: ["p1", "p2"],
|
||||||
|
explanations: EXPLANATIONS.map((e, i) => ({
|
||||||
|
id: `r1-${e.author}-x${1}`,
|
||||||
|
author: e.author,
|
||||||
|
cells: e.cells,
|
||||||
|
})),
|
||||||
|
});
|
||||||
|
|
||||||
|
const verificationOf = (off: string): Submission<"ablationVerification"> => ({
|
||||||
|
factors: [
|
||||||
|
{ factor: "p1", off: off === "p1", where: "solve.py:10-20" },
|
||||||
|
{ factor: "p2", off: off === "p2", where: "solve.py:30-40" },
|
||||||
|
],
|
||||||
|
});
|
||||||
|
|
||||||
|
/** 一份走完的研究：基线 0.96，第 1 轮采纳两因子的 r1-c1（1.25），消融两格测 1.10、1.20，收尾解封后停在导出。 */
|
||||||
|
function studyDir(t: TestContext): {
|
||||||
|
dir: string;
|
||||||
|
ledger: Ledger;
|
||||||
|
sealSeq: number;
|
||||||
|
} {
|
||||||
|
const root = tempDir(t);
|
||||||
|
const dir = join(root, "run");
|
||||||
|
const study = join(dir, "study");
|
||||||
|
mkdirSync(study, { recursive: true });
|
||||||
|
const ledger = Ledger.open(join(study, "ledger.sqlite"));
|
||||||
|
t.after(() => ledger.close());
|
||||||
|
const w = new Writer(ledger);
|
||||||
|
// 预算 4 次：基线、正向、两格消融，走完余额正好是 0。
|
||||||
|
ledger.append("config", {
|
||||||
|
config: { ...fullConfig(), runs: 4 },
|
||||||
|
task: facts as never,
|
||||||
|
});
|
||||||
|
|
||||||
|
const repo = join(study, "repo.git");
|
||||||
|
const work = join(root, "work");
|
||||||
|
createRepository(repo);
|
||||||
|
mkdirSync(work);
|
||||||
|
const version = (text: string) => {
|
||||||
|
writeFileSync(join(work, "solve.py"), text);
|
||||||
|
const result = saveVersion(repo, work, []);
|
||||||
|
assert.ok(result.saved);
|
||||||
|
return result.version;
|
||||||
|
};
|
||||||
|
|
||||||
|
ledger.append("code", {
|
||||||
|
object: "base",
|
||||||
|
version: version("value = 0\n"),
|
||||||
|
locations: null,
|
||||||
|
});
|
||||||
|
w.run("base-1", "baseline", 0, null, judged(0.96, "main"));
|
||||||
|
|
||||||
|
w.open("r1-proposal", "proposal", 1, ["a1", "a2"]);
|
||||||
|
const proposalSeq = w.sub(
|
||||||
|
"r1-proposal",
|
||||||
|
"a1",
|
||||||
|
"proposal",
|
||||||
|
null,
|
||||||
|
"r1-a1-1",
|
||||||
|
TWO_FACTORS,
|
||||||
|
);
|
||||||
|
w.close("r1-proposal");
|
||||||
|
w.decision("dedup", 1, null, {
|
||||||
|
fresh: [{ candidate: "r1-c1", proposals: ["r1-a1-1"] }],
|
||||||
|
carried: [],
|
||||||
|
});
|
||||||
|
w.protocol(
|
||||||
|
"r1-c1",
|
||||||
|
protocolOf("r1-c1", TWO_FACTORS, 1, "base", ["r1-a1-1"], ["c1", "c2"]),
|
||||||
|
);
|
||||||
|
|
||||||
|
w.open("r1-sp", "secondPrediction", 1, ["a1", "a2"]);
|
||||||
|
w.sub("r1-sp", "a1", "secondPrediction", "r1-c1", null, {
|
||||||
|
range: r(1.05, 1.15, 1.25),
|
||||||
|
body: "理由",
|
||||||
|
});
|
||||||
|
w.sub("r1-sp", "a2", "secondPrediction", "r1-c1", null, {
|
||||||
|
range: r(1.0, 1.1, 1.2),
|
||||||
|
body: "理由",
|
||||||
|
});
|
||||||
|
w.close("r1-sp");
|
||||||
|
const b = batch(1, 0.96, {
|
||||||
|
"r1-c1": { a1: r(1.05, 1.15, 1.25), a2: r(1.0, 1.1, 1.2) },
|
||||||
|
});
|
||||||
|
w.decision("batch", 1, b.input, b.output);
|
||||||
|
|
||||||
|
w.open("r1-implementation", "implementation", 1, ["x1"]);
|
||||||
|
w.sub("r1-implementation", "x1", "implementation", "r1-c1", null, {
|
||||||
|
factorLocations: [
|
||||||
|
{ factor: "p1", locations: ["solve.py:10-20"] },
|
||||||
|
{ factor: "p2", locations: ["solve.py:30-40"] },
|
||||||
|
],
|
||||||
|
safeguardLocations: [],
|
||||||
|
unassigned: [],
|
||||||
|
concerns: "无",
|
||||||
|
body: "实现说明",
|
||||||
|
});
|
||||||
|
ledger.append("code", {
|
||||||
|
object: "r1-c1",
|
||||||
|
version: version("value = 1\n"),
|
||||||
|
locations: null,
|
||||||
|
});
|
||||||
|
w.close("r1-implementation");
|
||||||
|
w.open("r1-verification-1", "verification", 1, ["v1"]);
|
||||||
|
w.sub("r1-verification-1", "v1", "verification", "r1-c1", null, {
|
||||||
|
clauses: [{ clause: "c1", found: true, where: "solve.py:10" }],
|
||||||
|
factors: [
|
||||||
|
{ factor: "p1", found: true, where: "solve.py:10-20" },
|
||||||
|
{ factor: "p2", found: true, where: "solve.py:30-40" },
|
||||||
|
],
|
||||||
|
safeguards: [],
|
||||||
|
unassigned: [],
|
||||||
|
body: "逐项核对",
|
||||||
|
});
|
||||||
|
w.close("r1-verification-1");
|
||||||
|
w.run("r1-forward-r1-c1-1", "forward", 1, "r1-c1", judged(1.25, "main"));
|
||||||
|
w.decision("adopt", 1, null, { adopted: "r1-c1" });
|
||||||
|
|
||||||
|
w.open("r1-explanation", "explanation", 1, ["a1", "a2"]);
|
||||||
|
for (const e of EXPLANATIONS) {
|
||||||
|
w.sub("r1-explanation", e.author, "explanation", null, null, {
|
||||||
|
explanations: [{ body: `机制（${e.author}）`, cells: e.cells }],
|
||||||
|
});
|
||||||
|
}
|
||||||
|
w.close("r1-explanation");
|
||||||
|
|
||||||
|
const cellInput: CellRuleInput = {
|
||||||
|
...modelInput(),
|
||||||
|
balance: 8,
|
||||||
|
batchCap: 2,
|
||||||
|
u: 0.5,
|
||||||
|
decay: 0.5,
|
||||||
|
seed: "seed",
|
||||||
|
};
|
||||||
|
const { rule, ...cellOutput } = runCellRule("all", cellInput);
|
||||||
|
ledger.append("decision", {
|
||||||
|
decision: "cells",
|
||||||
|
round: 1,
|
||||||
|
rule: rule.name,
|
||||||
|
ruleVersion: rule.version,
|
||||||
|
params: cellOutput.params,
|
||||||
|
seed: "seed",
|
||||||
|
input: cellInput as never,
|
||||||
|
output: cellOutput as never,
|
||||||
|
});
|
||||||
|
|
||||||
|
w.open(`r1-ablationImplementation-1`, "ablationImplementation", 1, ["x1"]);
|
||||||
|
for (const [key, text] of [
|
||||||
|
["p1", "value = 2\n"],
|
||||||
|
["p2", "value = 3\n"],
|
||||||
|
] as const) {
|
||||||
|
const cell = cellId("r1-c1", key);
|
||||||
|
w.sub(
|
||||||
|
`r1-ablationImplementation-1`,
|
||||||
|
"x1",
|
||||||
|
"ablationImplementation",
|
||||||
|
cell,
|
||||||
|
null,
|
||||||
|
{ alsoOff: [], cannotRun: null, concerns: "无" },
|
||||||
|
);
|
||||||
|
ledger.append("code", {
|
||||||
|
object: cell,
|
||||||
|
version: version(text),
|
||||||
|
locations: null,
|
||||||
|
});
|
||||||
|
}
|
||||||
|
w.close(`r1-ablationImplementation-1`);
|
||||||
|
w.open(`r1-ablationVerification-1`, "ablationVerification", 1, ["v1"]);
|
||||||
|
for (const key of ["p1", "p2"] as const) {
|
||||||
|
w.sub(
|
||||||
|
`r1-ablationVerification-1`,
|
||||||
|
"v1",
|
||||||
|
"ablationVerification",
|
||||||
|
cellId("r1-c1", key),
|
||||||
|
null,
|
||||||
|
verificationOf(key),
|
||||||
|
);
|
||||||
|
}
|
||||||
|
w.close(`r1-ablationVerification-1`);
|
||||||
|
for (const [key, score] of [
|
||||||
|
["p1", 1.1],
|
||||||
|
["p2", 1.2],
|
||||||
|
] as const) {
|
||||||
|
const cell = cellId("r1-c1", key);
|
||||||
|
const addRun = (id: string) =>
|
||||||
|
w.run(id, "ablation", 1, cell, judged(score, "main"));
|
||||||
|
addRun(`r1-ablation-${cell}-1`);
|
||||||
|
}
|
||||||
|
|
||||||
|
const settleInput: SettleInput = {
|
||||||
|
...modelInput(),
|
||||||
|
cells: [
|
||||||
|
{ factor: "p1", outcome: { kind: "valid", score: 1.1 } },
|
||||||
|
{ factor: "p2", outcome: { kind: "valid", score: 1.2 } },
|
||||||
|
],
|
||||||
|
};
|
||||||
|
const settled: Settlement = {
|
||||||
|
candidate: "r1-c1",
|
||||||
|
input: settleInput,
|
||||||
|
output: settleAblation(settleInput),
|
||||||
|
};
|
||||||
|
ledger.append("settlement", { round: 1, result: settled as never });
|
||||||
|
const betInput = { ...modelInput(), cells: settleInput.cells };
|
||||||
|
w.decision("bet", 1, betInput, chooseBet(betInput));
|
||||||
|
|
||||||
|
w.open("r1-wrapUp", "wrapUp", 1, ["a1", "a2"]);
|
||||||
|
w.sub("r1-wrapUp", "a1", "wrapUp", null, null, {
|
||||||
|
body: "甲的收尾：p1 撑了大半",
|
||||||
|
});
|
||||||
|
w.sub("r1-wrapUp", "a2", "wrapUp", null, null, { body: "乙的收尾" });
|
||||||
|
w.close("r1-wrapUp");
|
||||||
|
ledger.append("unseal", { candidate: "r1-c1", seals: [proposalSeq + 1] });
|
||||||
|
ledger.append("event", { event: "halted", reason: FULL_STOP, objects: [] });
|
||||||
|
return { dir, ledger, sealSeq: proposalSeq + 1 };
|
||||||
|
}
|
||||||
|
|
||||||
|
test("full 账目：一轮含消融的研究全部核对通过；撑几成、押的因子按手算对上", (t) => {
|
||||||
|
const { ledger } = studyDir(t);
|
||||||
|
const accounts = fullAccounts(ledger.records());
|
||||||
|
assert.deepEqual(accounts.confirmedStarts, {
|
||||||
|
total: 4,
|
||||||
|
baseline: 1,
|
||||||
|
forward: 1,
|
||||||
|
ablation: 2,
|
||||||
|
});
|
||||||
|
assert.deepEqual(accounts.balance, { charged: 4, reserved: 0, balance: 0 });
|
||||||
|
assert.ok(
|
||||||
|
Object.values(accounts.checks).every(Boolean),
|
||||||
|
JSON.stringify(accounts.checks),
|
||||||
|
);
|
||||||
|
const round = accounts.rounds[0]!;
|
||||||
|
assert.equal(round.adopted, "r1-c1");
|
||||||
|
assert.equal(round.adoptedScore, 1.25);
|
||||||
|
assert.ok(Math.abs(round.knownBestGap! - (2.635983 - 1.25)) < 1e-12);
|
||||||
|
assert.equal(round.cells.length, 2);
|
||||||
|
// 撑几成（方法.md 6.6）：(1.25 − 1.10) / 0.29 与 (1.25 − 1.20) / 0.29。
|
||||||
|
assert.ok(Math.abs(round.cells[0]!.support! - 0.15 / 0.29) < 1e-12);
|
||||||
|
assert.ok(Math.abs(round.cells[1]!.support! - 0.05 / 0.29) < 1e-12);
|
||||||
|
assert.equal(round.label!.label, "substitute"); // i = 0.690 − 1 < −0.2
|
||||||
|
assert.equal(round.bet!.decision, "bet");
|
||||||
|
assert.equal(round.bet!.factor, "p1");
|
||||||
|
assert.equal(round.unseal.pass, true);
|
||||||
|
assert.equal(round.explanationBeforeCells, true);
|
||||||
|
assert.deepEqual(
|
||||||
|
round.candidates[0]!.authorEstimates.map((u) => u.range),
|
||||||
|
[r(1.0, 1.1, 1.2)],
|
||||||
|
);
|
||||||
|
});
|
||||||
|
|
||||||
|
test("full 账目：缺解封、不跑的格被扣次都过不了核对", (t) => {
|
||||||
|
const { ledger } = studyDir(t);
|
||||||
|
const before = fullAccounts(ledger.records());
|
||||||
|
assert.equal(before.checks.unsealAfterWrapUp, true);
|
||||||
|
// 去掉解封记录重建一份账本太重，这里反过来验：另写一份没有解封的账。
|
||||||
|
const plain = ledger
|
||||||
|
.records()
|
||||||
|
.filter((x) => x.kind !== "unseal" && x.kind !== "event");
|
||||||
|
assert.equal(fullAccounts(plain).checks.unsealAfterWrapUp, false);
|
||||||
|
assert.equal(fullAccounts(plain).checks.haltedAtExport, false);
|
||||||
|
// "不跑"的格被扣次：给它补一条运行登记和启动，silentCellsNotCharged 应变 false。
|
||||||
|
ledger.append(
|
||||||
|
"event",
|
||||||
|
cellNotRunEvent(cellId("r1-c1", "p2"), "测试：核验不通过"),
|
||||||
|
);
|
||||||
|
ledger.append("runPlanned", {
|
||||||
|
run: "r1-ablation-r1-c1-p2-2",
|
||||||
|
purpose: "ablation",
|
||||||
|
round: 1,
|
||||||
|
object: "r1-c1-p2",
|
||||||
|
definition,
|
||||||
|
});
|
||||||
|
ledger.append("runStarted", { run: "r1-ablation-r1-c1-p2-2" });
|
||||||
|
const after = fullAccounts(ledger.records());
|
||||||
|
assert.equal(after.checks.silentCellsNotCharged, false);
|
||||||
|
assert.equal(after.checks.chargedEqualsStarts, true); // 账目本身仍平：5 次启动 5 次扣
|
||||||
|
});
|
||||||
|
|
||||||
|
test("full stats/export：从研究输出目录产出 stats.json/txt 和逐轮审阅材料", (t) => {
|
||||||
|
const { dir } = studyDir(t);
|
||||||
|
const statsOut = join(tempDir(t), "stats");
|
||||||
|
const stats = spawnSync(
|
||||||
|
process.execPath,
|
||||||
|
["eval/full/stats.ts", statsOut, dir],
|
||||||
|
{ encoding: "utf8" },
|
||||||
|
);
|
||||||
|
assert.equal(stats.status, 0, stats.stderr);
|
||||||
|
const text = readFileSync(join(statsOut, "stats.txt"), "utf8");
|
||||||
|
assert.match(text, /确认启动：基线 1、正向 1、消融 2，共 4；扣次 4，余额 0/);
|
||||||
|
assert.match(text, /采纳：r1-c1，分数 1\.250000，离已知最好 1\.385983/);
|
||||||
|
assert.match(
|
||||||
|
text,
|
||||||
|
/r1-c1-p1：核验 1 次、重做 0 次，跑没跑 跑了（扣次 1），实测 1\.100000，撑几成 0\.517/,
|
||||||
|
);
|
||||||
|
assert.match(text, /押的因子 p1/);
|
||||||
|
const summaries = JSON.parse(
|
||||||
|
readFileSync(join(statsOut, "stats.json"), "utf8"),
|
||||||
|
);
|
||||||
|
assert.equal(summaries[0].pass, true);
|
||||||
|
assert.equal(summaries[0].kill, null);
|
||||||
|
assert.equal(summaries[0].ratios.length, 4);
|
||||||
|
assert.ok(summaries[0].ratios.every((x: { flagged: boolean }) => !x.flagged));
|
||||||
|
|
||||||
|
const review = join(tempDir(t), "review");
|
||||||
|
const exported = spawnSync(
|
||||||
|
process.execPath,
|
||||||
|
["eval/full/export.ts", dir, review],
|
||||||
|
{ encoding: "utf8" },
|
||||||
|
);
|
||||||
|
assert.equal(exported.status, 0, exported.stderr);
|
||||||
|
const roundDir = join(review, "round-1");
|
||||||
|
assert.match(
|
||||||
|
readFileSync(join(roundDir, "explanations.txt"), "utf8"),
|
||||||
|
/机制（a1）/,
|
||||||
|
);
|
||||||
|
assert.match(
|
||||||
|
readFileSync(join(roundDir, "wrapups.txt"), "utf8"),
|
||||||
|
/甲的收尾：p1 撑了大半/,
|
||||||
|
);
|
||||||
|
assert.match(
|
||||||
|
readFileSync(join(roundDir, "proposals.txt"), "utf8"),
|
||||||
|
/多起点重启/,
|
||||||
|
);
|
||||||
|
assert.equal(
|
||||||
|
readFileSync(
|
||||||
|
join(roundDir, "cells", "r1-c1-p1", "full", "solve.py"),
|
||||||
|
"utf8",
|
||||||
|
),
|
||||||
|
"value = 1\n",
|
||||||
|
);
|
||||||
|
assert.equal(
|
||||||
|
readFileSync(
|
||||||
|
join(roundDir, "cells", "r1-c1-p1", "cell", "solve.py"),
|
||||||
|
"utf8",
|
||||||
|
),
|
||||||
|
"value = 2\n",
|
||||||
|
);
|
||||||
|
const verifications = JSON.parse(
|
||||||
|
readFileSync(
|
||||||
|
join(roundDir, "cells", "r1-c1-p1", "verifications.json"),
|
||||||
|
"utf8",
|
||||||
|
),
|
||||||
|
);
|
||||||
|
assert.equal(verifications[0].outcome.passed, true);
|
||||||
|
const settlement = JSON.parse(
|
||||||
|
readFileSync(join(roundDir, "settlement.json"), "utf8"),
|
||||||
|
);
|
||||||
|
assert.equal(settlement.bet.content.output.factor, "p1");
|
||||||
|
assert.match(readFileSync(join(roundDir, "review.txt"), "utf8"), /第3样/);
|
||||||
|
});
|
||||||
|
|
||||||
|
test("full CPU/墙钟比：测不到或没启动记 null，正常退出算出比值", () => {
|
||||||
|
assert.equal(cpuRatio({ process: { ending: "notStarted" } }), null);
|
||||||
|
assert.ok(Math.abs(cpuRatio({ process: exited })! - 100 / 101) < 1e-12);
|
||||||
|
assert.equal(
|
||||||
|
cpuRatio({
|
||||||
|
process: {
|
||||||
|
ending: "exited",
|
||||||
|
cpuSeconds: { status: "unmeasured" },
|
||||||
|
wallSeconds: { status: "measured", value: 1 },
|
||||||
|
},
|
||||||
|
}),
|
||||||
|
null,
|
||||||
|
);
|
||||||
|
});
|
||||||
Reference in New Issue
Block a user