iomgaa
0ae495f8e8
- eval/full/run.ts：配置照 eval/round-zero/research.json 只换编号，真实跑到停下； launches/machine/timeline/outcome 照任务书 16 惯例；--kill-after-first-ablation-start 在第一个消融运行确认启动后、结果落账前 SIGKILL 控制进程，恢复续跑到底， kill-report.json 写三条检查（同一容器不重跑、只扣一次、结果照常写入）。 - eval/lib/full.ts：账目核对 fullAccounts——逐轮走向（候选分数、采纳、选格、每格 核验重做次数与扣次、结算、押的因子）加 12 项程序账目检查（阶段开关成对、 扣次=确认启动数、余额为 0、解释冻结早于首格启动、每轮最多一条结算、解封在 收尾关闭之后且覆盖全部选中候选、跑不起来/不跑的格零扣次、第 2 轮起基线链 重算、停在"导出还没写"等）与 CPU/墙钟比（<0.9 列出并附同时段负载，只列事实）。 - eval/full/stats.ts、export.ts：统计输出与按轮按格的审阅材料导出。 - 不沿用 eval/forward/acceptance.ts 的"启动区间互不重叠"（这次 3 份同时跑）。 - tests/full-eval.test.ts 4 条：真实账本 + rules 真实计算走完一轮含消融的研究， 撑几成与配合关系标签按方法.md 6.6 手算对照。npm run check 通过，npm test 476 条通过。
377 lines
14 KiB
TypeScript
377 lines
14 KiB
TypeScript
import type { LedgerRecord } from "../../src/ledger/index.ts";
|
|
import {
|
|
Board,
|
|
CELL_NOT_RUN,
|
|
cellId,
|
|
FORWARD_NOT_RUN,
|
|
ORIGINAL_BASELINE,
|
|
validScore,
|
|
type StudyRecord,
|
|
} from "../../src/materials/index.ts";
|
|
import { improvement, runBalance } from "../../src/rules/index.ts";
|
|
import { ablationOutcome } from "../../src/submissions/index.ts";
|
|
import { checkTaskFacts } from "../../src/tasks/index.ts";
|
|
import { frozenConfig, scoreBand } from "./forward.ts";
|
|
import { ofKind } from "./ledger.ts";
|
|
|
|
// 完整研究（任务书 17 第三块）的账目核对：只从账本记录重算事实，不作质量判断。
|
|
// 质量判断（解释好不好、核验判得对不对、收尾写得怎么样）由父会话派全新上下文的子 Agent 做。
|
|
|
|
/** 研究走完一轮接一轮、余额用完后的停点（任务书 17 第 2.1 节）。 */
|
|
export const FULL_STOP = "导出还没写";
|
|
|
|
/** CPU 时间 / 墙钟时间低于这个比值就列出来和机器负载放在一起（任务书 17 第 3.1 节"明显低于 1"；只列事实）。 */
|
|
export const CPU_RATIO_FLAG = 0.9;
|
|
|
|
/** 一份完整研究的账目：每轮的走向 + 全研究的程序账目核对（任务书 17 第 3.2 节第 6、7 样的可计算部分）。 */
|
|
export function fullAccounts(records: LedgerRecord[]) {
|
|
const board = new Board(records);
|
|
const config = frozenConfig(records);
|
|
// SAFETY: 账本 config 记录的内容形状由 materials 的 StudyRecord 定义（docs/结构.md 第八部分），ledger 里只存 Json。
|
|
const studyRecord = ofKind(records, "config")[0]!
|
|
.content as unknown as StudyRecord;
|
|
const facts = checkTaskFacts(studyRecord.task);
|
|
const events = ofKind(records, "event").map((r) => ({
|
|
...r.content,
|
|
seq: r.seq,
|
|
}));
|
|
const started = ofKind(records, "runStarted");
|
|
const runs = ofKind(records, "runPlanned").map((p) => ({
|
|
plan: p,
|
|
starts: started.filter((s) => s.content.run === p.content.run),
|
|
results: ofKind(records, "runResult").filter(
|
|
(r) => r.content.run === p.content.run,
|
|
),
|
|
}));
|
|
const balance = runBalance(config.runs, board.runStates());
|
|
|
|
// 阶段开关成对：每个开恰有一个比它晚的关；没有没开的关。
|
|
const opens = ofKind(records, "stageOpen");
|
|
const closes = ofKind(records, "stageClose");
|
|
const stages = opens.map((r) => {
|
|
const closing = closes.filter(
|
|
(c) => c.content.stageId === r.content.stageId,
|
|
);
|
|
return {
|
|
...r.content,
|
|
openSeq: r.seq,
|
|
closeSeq: closing[0]?.seq ?? null,
|
|
pass: closing.length === 1 && closing[0]!.seq > r.seq,
|
|
};
|
|
});
|
|
const orphanCloses = closes.filter(
|
|
(c) => !opens.some((o) => o.content.stageId === c.content.stageId),
|
|
);
|
|
const closeSeqOf = (stageId: string) =>
|
|
stages.find((s) => s.stageId === stageId)?.closeSeq ?? null;
|
|
|
|
const roundIds = [
|
|
...new Set([
|
|
...opens.map((o) => o.content.round),
|
|
...ofKind(records, "decision").map((d) => d.content.round),
|
|
...ofKind(records, "settlement").map((s) => s.content.round),
|
|
]),
|
|
]
|
|
.filter((r) => r >= 1)
|
|
.sort((a, b) => a - b);
|
|
|
|
const rounds = roundIds.map((round) => {
|
|
const batch = board.decision("batch", round);
|
|
const chosen = batch?.output.chosen ?? [];
|
|
// 每个入选候选一行（任务书 17 第 3.2 节第 6 样）：路线代号不自动猜，留给人工填。
|
|
const candidates = chosen.map((candidate) => {
|
|
const run = board.runOf(candidate);
|
|
const score = validScore(run?.result?.judged ?? null);
|
|
const merged =
|
|
batch!.output.candidates.find((c) => c.id === candidate)?.merged ??
|
|
null;
|
|
return {
|
|
candidate,
|
|
score,
|
|
merged,
|
|
authorEstimates: board
|
|
.unsealed()
|
|
.filter((u) => u.candidate === candidate),
|
|
band: merged === null ? null : scoreBand(score, merged),
|
|
started: run !== undefined,
|
|
notRunEvents: events.filter(
|
|
(e) => e.event === FORWARD_NOT_RUN && e.objects.includes(candidate),
|
|
),
|
|
route: "路线代号由父会话审阅时人工填，不自动按关键词贴",
|
|
};
|
|
});
|
|
const adopt = board.decision("adopt", round) ?? null;
|
|
const adopted = adopt?.output.adopted ?? null;
|
|
const adoptedScore =
|
|
adopted === null
|
|
? null
|
|
: validScore(board.runOf(adopted)?.result?.judged ?? null);
|
|
const bestSoFar =
|
|
board.decision("batch", round) === undefined ? null : null;
|
|
void bestSoFar;
|
|
const knownBestGap =
|
|
adoptedScore === null || facts.knownBest === null
|
|
? null
|
|
: improvement(
|
|
adoptedScore,
|
|
facts.knownBest,
|
|
facts.finalMetric.direction,
|
|
);
|
|
|
|
// 消融（任务书 17 第 3.2 节第 4 样的可计算部分和第 7 样的逐格账目）。
|
|
const cellsDecision = board.decision("cells", round) ?? null;
|
|
const settlement = board.settlement(round) ?? null;
|
|
const bet = board.decision("bet", round) ?? null;
|
|
const cells = (cellsDecision?.output.chosen ?? []).map((key) => {
|
|
const cell = cellId(adopted!, key);
|
|
// published 只收已关阶段的最后一版；研究走完了，这些阶段都已关闭。
|
|
const impls = board
|
|
.published("ablationImplementation")
|
|
.filter((s) => s.target === cell);
|
|
const lastImpl = impls.at(-1) ?? null;
|
|
const verifications = board
|
|
.published("ablationVerification")
|
|
.filter((s) => s.target === cell);
|
|
const lastVerification = verifications.at(-1) ?? null;
|
|
const lastOutcome =
|
|
lastVerification === null || lastImpl === null
|
|
? null
|
|
: ablationOutcome(
|
|
lastVerification.content,
|
|
key,
|
|
lastImpl.content.alsoOff,
|
|
);
|
|
const cellRuns = runs.filter(
|
|
(r) =>
|
|
r.plan.content.purpose === "ablation" &&
|
|
r.plan.content.round === round &&
|
|
r.plan.content.object === cell,
|
|
);
|
|
const lastRun = cellRuns.at(-1) ?? null;
|
|
const settled =
|
|
settlement?.output.cells.find((c) => c.factor === key) ?? null;
|
|
const settleInput =
|
|
settlement?.input.cells.find((c) => c.factor === key)?.outcome ?? null;
|
|
const notRun = events.filter(
|
|
(e) => e.event === CELL_NOT_RUN && e.objects.includes(cell),
|
|
);
|
|
const cannotRun = lastImpl === null ? null : lastImpl.content.cannotRun;
|
|
return {
|
|
cell,
|
|
factor: key,
|
|
implementationCount: impls.length,
|
|
redoCount: Math.max(0, impls.length - 1),
|
|
verificationCount: verifications.length,
|
|
cannotRun,
|
|
notRunEvents: notRun,
|
|
lastVerificationPassed:
|
|
lastOutcome === null ? null : lastOutcome.passed,
|
|
charged: cellRuns.filter((r) => r.starts.length > 0).length,
|
|
ran:
|
|
lastRun !== null &&
|
|
lastRun.results.length === 1 &&
|
|
lastRun.results[0]!.content.process.ending !== "notStarted",
|
|
measured:
|
|
settleInput !== null &&
|
|
(settleInput.kind === "valid" || settleInput.kind === "joint")
|
|
? settleInput.score
|
|
: null,
|
|
alsoOff:
|
|
settleInput !== null && settleInput.kind === "joint"
|
|
? settleInput.off
|
|
: [],
|
|
outcome: settled?.outcome ?? null,
|
|
support: settled?.support ?? null,
|
|
merged: settled?.merged ?? null,
|
|
worldBits: settled?.worldBits ?? null,
|
|
surprise: settled?.surprise ?? null,
|
|
// 每个选中的格都已结束这一段：交了"跑不起来"，或核验不通过有"不跑"事件，或最近一次核验通过且跑出了结果。
|
|
accounted:
|
|
lastImpl !== null &&
|
|
(cannotRun !== null ||
|
|
notRun.length > 0 ||
|
|
(lastOutcome?.passed === true &&
|
|
lastRun !== null &&
|
|
lastRun.results.length === 1 &&
|
|
lastRun.results[0]!.content.process.ending !== "notStarted")),
|
|
};
|
|
});
|
|
|
|
// 解释阶段关闭早于本轮任何一格的确认启动（任务书 17 第 3.2 节第 7 样）。
|
|
const explanationClose = closeSeqOf(`r${round}-explanation`);
|
|
const cellStartSeqs = runs
|
|
.filter(
|
|
(r) =>
|
|
r.plan.content.purpose === "ablation" &&
|
|
r.plan.content.round === round,
|
|
)
|
|
.flatMap((r) => r.starts.map((s) => s.seq));
|
|
const firstCellStart =
|
|
cellStartSeqs.length === 0 ? null : Math.min(...cellStartSeqs);
|
|
const explanationBeforeCells =
|
|
cellsDecision === null
|
|
? null
|
|
: explanationClose !== null &&
|
|
(firstCellStart === null || explanationClose < firstCellStart);
|
|
|
|
// 轮末解封：本轮选中的每个候选都有解封记录，且写在收尾阶段关闭之后（任务书 17 第 3.2 节第 7 样）。
|
|
const wrapUpClose = closeSeqOf(`r${round}-wrapUp`);
|
|
const unseals = ofKind(records, "unseal").filter((u) =>
|
|
chosen.includes(u.content.candidate),
|
|
);
|
|
const unseal = {
|
|
wrapUpClose,
|
|
perCandidate: chosen.map((candidate) => {
|
|
const u = unseals.find((x) => x.content.candidate === candidate);
|
|
return {
|
|
candidate,
|
|
unsealSeq: u?.seq ?? null,
|
|
afterWrapUpClose:
|
|
u !== undefined && wrapUpClose !== null && u.seq > wrapUpClose,
|
|
};
|
|
}),
|
|
pass:
|
|
wrapUpClose !== null &&
|
|
chosen.length > 0 &&
|
|
chosen.every((candidate) => {
|
|
const u = unseals.find((x) => x.content.candidate === candidate);
|
|
return u !== undefined && u.seq > wrapUpClose;
|
|
}),
|
|
};
|
|
|
|
return {
|
|
round,
|
|
baseline: board.baselineOf(round),
|
|
candidates,
|
|
adopted,
|
|
adoptedScore,
|
|
knownBestGap,
|
|
hasAblation: cellsDecision !== null,
|
|
cellsDecision,
|
|
cells,
|
|
label: settlement?.output.label ?? null,
|
|
judgments: settlement?.output.judgments ?? null,
|
|
inseparable: settlement?.output.inseparable ?? null,
|
|
finding: settlement?.output.finding ?? null,
|
|
bet:
|
|
bet === null
|
|
? null
|
|
: {
|
|
decision: bet.output.decision,
|
|
factor: bet.output.factor,
|
|
predictedGain: bet.output.predictedGain,
|
|
offBetter: bet.output.offBetter,
|
|
},
|
|
settlementCount: board.settlements().filter((s) => s.round === round)
|
|
.length,
|
|
explanationBeforeCells,
|
|
unseal,
|
|
};
|
|
});
|
|
|
|
// 第 2 轮起的基线：从采纳决定和结算的"消融中发现"逐轮重算，和 Board 现算的对照；
|
|
// 再核对第 r 轮冻结的协议都写在当时的基线上（方法.md 4.3 ①）。
|
|
const chain: Array<{ round: number; baseline: string }> = [
|
|
{ round: 1, baseline: ORIGINAL_BASELINE },
|
|
];
|
|
for (const round of roundIds) {
|
|
const adopted = board.decision("adopt", round)?.output.adopted ?? null;
|
|
const found = board.settlement(round)?.output.finding.chosen ?? null;
|
|
const previous = chain.at(-1)!.baseline;
|
|
chain.push({
|
|
round: round + 1,
|
|
baseline:
|
|
found === null
|
|
? adopted === null
|
|
? previous
|
|
: adopted
|
|
: cellId(adopted!, found),
|
|
});
|
|
}
|
|
const baselineChain = {
|
|
chain,
|
|
boardAgrees: chain.every((c) => board.baselineOf(c.round) === c.baseline),
|
|
protocolsOnBaseline: board
|
|
.protocols()
|
|
.filter((p) => p.protocol.round >= 2)
|
|
.every(
|
|
(p) =>
|
|
p.protocol.baseline ===
|
|
chain.find((c) => c.round === p.protocol.round)?.baseline,
|
|
),
|
|
};
|
|
|
|
// "跑不起来"和"不跑"的格不扣次（任务书 17 第 3.2 节第 7 样）：这些格没有任何运行登记。
|
|
const silentCells = rounds.flatMap((r) =>
|
|
r.cells.filter((c) => c.cannotRun !== null || c.notRunEvents.length > 0),
|
|
);
|
|
const silentNotCharged = silentCells.every((c) => c.charged === 0);
|
|
|
|
const halt = events.filter((e) => e.event === "halted").at(-1) ?? null;
|
|
const checks = {
|
|
stagesPaired: stages.every((s) => s.pass) && orphanCloses.length === 0,
|
|
chargedEqualsStarts: balance.charged === started.length,
|
|
finalBalanceZero: balance.balance === 0,
|
|
reservedZero: balance.reserved === 0,
|
|
runsFinished: runs.length > 0 && runs.every((r) => r.results.length === 1),
|
|
explanationBeforeCells: rounds.every(
|
|
(r) => r.explanationBeforeCells !== false,
|
|
),
|
|
oneSettlementPerRound: rounds.every((r) => r.settlementCount <= 1),
|
|
unsealAfterWrapUp: rounds.every((r) => r.unseal.pass),
|
|
cellsAccounted: rounds.every((r) => r.cells.every((c) => c.accounted)),
|
|
silentCellsNotCharged: silentNotCharged,
|
|
baselineChain:
|
|
baselineChain.boardAgrees && baselineChain.protocolsOnBaseline,
|
|
haltedAtExport:
|
|
halt !== null &&
|
|
halt.reason === FULL_STOP &&
|
|
halt.seq === records.at(-1)!.seq,
|
|
};
|
|
return {
|
|
study: config.studyId,
|
|
initial: config.runs,
|
|
knownBest: facts.knownBest,
|
|
confirmedStarts: {
|
|
total: started.length,
|
|
baseline: runs.filter(
|
|
(r) => r.plan.content.purpose === "baseline" && r.starts.length > 0,
|
|
).length,
|
|
forward: runs.filter(
|
|
(r) => r.plan.content.purpose === "forward" && r.starts.length > 0,
|
|
).length,
|
|
ablation: runs.filter(
|
|
(r) => r.plan.content.purpose === "ablation" && r.starts.length > 0,
|
|
).length,
|
|
},
|
|
balance,
|
|
runs,
|
|
rounds,
|
|
baselineChain,
|
|
stages,
|
|
orphanCloses,
|
|
halt,
|
|
checks,
|
|
};
|
|
}
|
|
|
|
/** 一次正式运行的 CPU 时间 / 墙钟时间（任务书 17 第 3.1 节）；测不到或没启动为 null。 */
|
|
export function cpuRatio(result: {
|
|
process: {
|
|
ending: string;
|
|
cpuSeconds?: { status: string; value?: number };
|
|
wallSeconds?: { status: string; value?: number };
|
|
};
|
|
}): number | null {
|
|
const p = result.process;
|
|
if (p.ending === "notStarted") return null;
|
|
if (
|
|
p.cpuSeconds?.status !== "measured" ||
|
|
p.wallSeconds?.status !== "measured"
|
|
)
|
|
return null;
|
|
return p.wallSeconds.value === 0
|
|
? null
|
|
: p.cpuSeconds.value! / p.wallSeconds.value!;
|
|
}
|