iomgaa
0ae495f8e8
- eval/full/run.ts：配置照 eval/round-zero/research.json 只换编号，真实跑到停下； launches/machine/timeline/outcome 照任务书 16 惯例；--kill-after-first-ablation-start 在第一个消融运行确认启动后、结果落账前 SIGKILL 控制进程，恢复续跑到底， kill-report.json 写三条检查（同一容器不重跑、只扣一次、结果照常写入）。 - eval/lib/full.ts：账目核对 fullAccounts——逐轮走向（候选分数、采纳、选格、每格 核验重做次数与扣次、结算、押的因子）加 12 项程序账目检查（阶段开关成对、 扣次=确认启动数、余额为 0、解释冻结早于首格启动、每轮最多一条结算、解封在 收尾关闭之后且覆盖全部选中候选、跑不起来/不跑的格零扣次、第 2 轮起基线链 重算、停在"导出还没写"等）与 CPU/墙钟比（<0.9 列出并附同时段负载，只列事实）。 - eval/full/stats.ts、export.ts：统计输出与按轮按格的审阅材料导出。 - 不沿用 eval/forward/acceptance.ts 的"启动区间互不重叠"（这次 3 份同时跑）。 - tests/full-eval.test.ts 4 条：真实账本 + rules 真实计算走完一轮含消融的研究， 撑几成与配合关系标签按方法.md 6.6 手算对照。npm run check 通过，npm test 476 条通过。
274 lines
12 KiB
TypeScript
274 lines
12 KiB
TypeScript
import { mkdirSync, writeFileSync } from "node:fs";
|
||
import { join, resolve } from "node:path";
|
||
import {
|
||
Board,
|
||
cellId,
|
||
explanationId,
|
||
protocolMetrics,
|
||
splitCell,
|
||
type TaskMaterial,
|
||
} from "../../src/materials/index.ts";
|
||
import { ablationOutcome } from "../../src/submissions/index.ts";
|
||
import { checkoutVersion, compareVersions } from "../../src/workspace/index.ts";
|
||
import { jsonFile } from "../lib/forward.ts";
|
||
import { finalSubmissions, ofKind, readLedger } from "../lib/ledger.ts";
|
||
|
||
// 任务书 17 第 3.2 节研究质量评估的审阅材料：解释、选格决定、消融格（协议、位置表、代码与 diff、实现与核验往来）、
|
||
// 结算、收尾、下一轮提案。只导出事实和原文，不替审阅者下结论。
|
||
// node eval/full/export.ts <研究输出目录> <全新绝对审阅目录>
|
||
const [dir, output, ...extra] = process.argv.slice(2);
|
||
if (!dir || !output || extra.length || output !== resolve(output)) {
|
||
throw new Error(
|
||
"用法：node eval/full/export.ts <研究输出目录> <全新绝对审阅目录>",
|
||
);
|
||
}
|
||
mkdirSync(output); // 不覆盖以前的审阅材料。
|
||
const study = join(dir, "study");
|
||
const records = readLedger(study);
|
||
const board = new Board(records);
|
||
const repo = join(study, "repo.git");
|
||
const finals = finalSubmissions(records);
|
||
const roundOf = (stageId: string) => board.stage(stageId).round;
|
||
const roundIds = [
|
||
...new Set(ofKind(records, "decision").map((d) => d.content.round)),
|
||
]
|
||
.filter((r) => r >= 1)
|
||
.sort((a, b) => a - b);
|
||
|
||
const checkout = (version: string, path: string) => {
|
||
mkdirSync(path);
|
||
checkoutVersion(repo, version, path);
|
||
};
|
||
const json = (v: unknown) => `${JSON.stringify(v, null, 2)}\n`;
|
||
|
||
const index = [];
|
||
for (const round of roundIds) {
|
||
const roundDir = join(output, `round-${round}`);
|
||
mkdirSync(roundDir);
|
||
|
||
// 本轮提案（下一轮提案在 round-<r+1> 里，审阅"收尾→接力"时把两份对着读）。
|
||
const proposals = finals.filter(
|
||
(s) =>
|
||
s.content.kind === "proposal" &&
|
||
roundOf(s.content.stageId) === round,
|
||
);
|
||
jsonFile(join(roundDir, "proposals.json"), proposals);
|
||
writeFileSync(
|
||
join(roundDir, "proposals.txt"),
|
||
proposals
|
||
.map(
|
||
(s) =>
|
||
`${"=".repeat(20)} ${s.content.item}（${s.content.author}，记录 ${s.seq}）\n${json(s.content.content)}`,
|
||
)
|
||
.join("\n"),
|
||
);
|
||
|
||
// 解释（任务书 17 第 3.2 节第 1 样）：作者、正文、每格三个数。
|
||
const explanations = finals.filter(
|
||
(s) =>
|
||
s.content.kind === "explanation" &&
|
||
roundOf(s.content.stageId) === round,
|
||
);
|
||
jsonFile(join(roundDir, "explanations.json"), explanations);
|
||
writeFileSync(
|
||
join(roundDir, "explanations.txt"),
|
||
explanations
|
||
.flatMap((s) =>
|
||
(
|
||
s.content.content as {
|
||
explanations: Array<{
|
||
body: string;
|
||
cells?: Array<{ factor: string; range: unknown }>;
|
||
}>;
|
||
}
|
||
).explanations.map(
|
||
(e, i) =>
|
||
`${"=".repeat(20)} ${explanationId(round, s.content.author, i + 1)}（${s.content.author} 的第 ${i + 1} 条，记录 ${s.seq}）\n${e.body}\n三个数：${(e.cells ?? []).map((c) => `${c.factor} ${JSON.stringify(c.range)}`).join("；")}`,
|
||
),
|
||
)
|
||
.join("\n\n"),
|
||
);
|
||
|
||
// 选格决定全文（第 2 样）：因子、合并范围、各组合的信息价值/成本/净值、选中与没选的原因都在这条决定里。
|
||
const cellsDecision =
|
||
ofKind(records, "decision").find(
|
||
(d) => d.content.decision === "cells" && d.content.round === round,
|
||
) ?? null;
|
||
const chosenCells = board.decision("cells", round)?.output.chosen ?? null;
|
||
jsonFile(join(roundDir, "cells-decision.json"), cellsDecision);
|
||
|
||
// 结算全文（第 4 样）：结算记录和同一事务里押的因子决定。
|
||
const settlement =
|
||
ofKind(records, "settlement").find((s) => s.content.round === round) ??
|
||
null;
|
||
const bet =
|
||
ofKind(records, "decision").find(
|
||
(d) => d.content.decision === "bet" && d.content.round === round,
|
||
) ?? null;
|
||
jsonFile(join(roundDir, "settlement.json"), { settlement, bet });
|
||
|
||
// 收尾全文（第 5 样）。
|
||
const wrapUps = finals.filter(
|
||
(s) =>
|
||
s.content.kind === "wrapUp" && roundOf(s.content.stageId) === round,
|
||
);
|
||
jsonFile(join(roundDir, "wrapups.json"), wrapUps);
|
||
writeFileSync(
|
||
join(roundDir, "wrapups.txt"),
|
||
wrapUps
|
||
.map(
|
||
(s) =>
|
||
`${"=".repeat(20)} ${s.content.author}（记录 ${s.seq}）\n${(s.content.content as { body: string }).body}`,
|
||
)
|
||
.join("\n\n"),
|
||
);
|
||
|
||
// 消融格（第 3 样）：冻结协议、位置表、完整代码与格代码及 diff、alsoOff、concerns、每次核验逐项结论与退回、重做记录。
|
||
const cells = [];
|
||
if (chosenCells !== null) {
|
||
const adopt = board.decision("adopt", round)!;
|
||
const candidate = adopt.output.adopted!;
|
||
const full = board.code(candidate);
|
||
const baseline = board.code(board.baselineOf(round));
|
||
for (const key of chosenCells) {
|
||
const cell = cellId(candidate, key);
|
||
const cellDir = join(roundDir, "cells", cell);
|
||
mkdirSync(cellDir, { recursive: true });
|
||
jsonFile(join(cellDir, "protocol.json"), {
|
||
record: ofKind(records, "protocol")
|
||
.filter((p) => p.content.candidate === candidate)
|
||
.at(-1),
|
||
resolvedMetrics: protocolMetrics(
|
||
board,
|
||
board.protocol(candidate),
|
||
),
|
||
factorLocations: full.locations,
|
||
baselineVersion: baseline.version,
|
||
fullVersion: full.version,
|
||
});
|
||
checkout(full.version, join(cellDir, "full"));
|
||
checkout(baseline.version, join(cellDir, "baseline"));
|
||
// 每次实现和代码版本对号（同一事务里先写提交、紧接着写代码；交"跑不起来"时没有代码记录）。
|
||
const impls = finals.filter(
|
||
(s) =>
|
||
s.content.kind === "ablationImplementation" &&
|
||
s.content.target === cell,
|
||
);
|
||
const codes = ofKind(records, "code").filter(
|
||
(c) => c.content.object === cell,
|
||
);
|
||
const attempts = impls.map((s) => ({
|
||
submission: s,
|
||
code: codes.find((c) => c.seq === s.seq + 1) ?? null,
|
||
}));
|
||
jsonFile(join(cellDir, "implementations.json"), attempts);
|
||
const verifications = finals
|
||
.filter(
|
||
(s) =>
|
||
s.content.kind === "ablationVerification" &&
|
||
s.content.target === cell,
|
||
)
|
||
.map((s) => {
|
||
const impl =
|
||
impls.filter((x) => x.seq < s.seq).at(-1) ?? null;
|
||
return {
|
||
submission: s,
|
||
judgedImplSeq: impl?.seq ?? null,
|
||
outcome:
|
||
impl === null
|
||
? null
|
||
: ablationOutcome(
|
||
s.content.content as Parameters<
|
||
typeof ablationOutcome
|
||
>[0],
|
||
key,
|
||
(
|
||
impl.content.content as {
|
||
alsoOff: string[];
|
||
}
|
||
).alsoOff,
|
||
),
|
||
};
|
||
});
|
||
jsonFile(join(cellDir, "verifications.json"), verifications);
|
||
jsonFile(
|
||
join(cellDir, "rejections.json"),
|
||
ofKind(records, "rejection").filter(
|
||
(r) => r.content.target === cell,
|
||
),
|
||
);
|
||
const lastCode = codes.at(-1) ?? null;
|
||
if (lastCode !== null) {
|
||
checkout(lastCode.content.version, join(cellDir, "cell"));
|
||
jsonFile(
|
||
join(cellDir, "diff.json"),
|
||
compareVersions(
|
||
repo,
|
||
full.version,
|
||
lastCode.content.version,
|
||
),
|
||
);
|
||
}
|
||
for (const attempt of attempts) {
|
||
if (attempt.code === null) continue;
|
||
const path = join(cellDir, `code-${attempt.code.seq}`);
|
||
checkout(attempt.code.content.version, path);
|
||
jsonFile(
|
||
join(cellDir, `diff-${attempt.code.seq}.json`),
|
||
compareVersions(
|
||
repo,
|
||
full.version,
|
||
attempt.code.content.version,
|
||
),
|
||
);
|
||
}
|
||
jsonFile(
|
||
join(cellDir, "calls.json"),
|
||
ofKind(records, "callStarted")
|
||
.filter((c) => {
|
||
const material = c.content.materials as TaskMaterial;
|
||
return (
|
||
material.object === cell &&
|
||
[
|
||
"ablationImplementation",
|
||
"ablationVerification",
|
||
].includes(material.kind)
|
||
);
|
||
})
|
||
.map((c) => ({
|
||
record: c,
|
||
sessions: join(
|
||
study,
|
||
"calls",
|
||
c.content.call,
|
||
"sessions",
|
||
),
|
||
})),
|
||
);
|
||
cells.push({
|
||
cell,
|
||
factor: splitCell(cell).key,
|
||
dir: cellDir,
|
||
ran: lastCode !== null,
|
||
});
|
||
}
|
||
}
|
||
|
||
writeFileSync(
|
||
join(roundDir, "review.txt"),
|
||
`任务书17第3.2节，只读本目录，中文直接回复，不写文件。\n` +
|
||
`第1样（解释）：读 explanations.txt、cells-decision.json、settlement.json。看不到消融结果地回答：机制是否具体、写了什么情况下不成立没有、三个数和正文是否一致、范围宽窄有没有依据、同轮几人的解释有没有真正不同、"没有具体机制判断"用得诚不诚实；再对 settlement.json 看每条解释在每格被确认还是否定。\n` +
|
||
`第2样（选格）：读 cells-decision.json，列出各格合并范围、各组合信息价值/成本/净值、选中和没选的原因。\n` +
|
||
`第3样（消融实现与核验，每格一个全新上下文）：读 cells/<格>/ 下的 protocol.json、full/、cell/、diff.json、implementations.json、verifications.json、rejections.json。判断：是不是只关了指定的因子；保障原样保留没有、关掉的因子的指标写 null 没有；答案路径标签和关掉以后的分支对不对得上；核验员每次逐项结论判得对不对；重做的只改了被指出的地方没有。\n` +
|
||
`第5样（收尾与接力）：读 wrapups.txt、settlement.json，再对着 round-${round + 1}/proposals.txt（没有就是最后一轮）看下一轮提案用上了哪些发现。\n`,
|
||
);
|
||
index.push({
|
||
round,
|
||
dir: roundDir,
|
||
hasAblation: chosenCells !== null,
|
||
cells,
|
||
});
|
||
}
|
||
jsonFile(join(output, "index.json"), { study: dir, rounds: index });
|
||
console.log(`审阅材料：${output}`);
|