iomgaa
de8e13b031
审阅发现spawn后快照早于恢复实际执行，不能证明没有重启；改用杀前至目标结果落账的Docker历史事件，保留缓存完整性限制。逐份stats不再冒称任务书总体通过；独立验收要求三份不同研究、成功杀进程及不重叠区间。两项缺陷附回归，7项定向测试与check通过，独立复审无新问题；真实B尚待启动。
241 lines
8.3 KiB
TypeScript
241 lines
8.3 KiB
TypeScript
import { existsSync, mkdirSync, readFileSync } from "node:fs";
|
||
import { join, resolve } from "node:path";
|
||
import type { LedgerRecord } from "../../src/ledger/index.ts";
|
||
import {
|
||
Board,
|
||
validScore,
|
||
type TaskMaterial,
|
||
} from "../../src/materials/index.ts";
|
||
import {
|
||
forwardAccounts,
|
||
frozenConfig,
|
||
jsonFile,
|
||
scoreBand,
|
||
} from "../lib/forward.ts";
|
||
import {
|
||
sameContainerThroughResult,
|
||
type ContainerEvent,
|
||
type ContainerSnapshot,
|
||
type LifecycleWindow,
|
||
} from "../lib/forward-evidence.ts";
|
||
import { finalSubmissions, ofKind, readLedger } from "../lib/ledger.ts";
|
||
import { lightCheckUsage } from "../lib/light-checks.ts";
|
||
import { referenceScores } from "../lib/reference-scores.ts";
|
||
|
||
// 五项表中可计算的字段。路线与审阅结论故意不自动猜，由父会话按候选审阅包填写报告。
|
||
// node eval/forward/stats.ts <全新绝对输出目录> <五份参考副本根目录> <检验目录>...
|
||
const [output, referenceRoot, ...dirs] = process.argv.slice(2);
|
||
if (
|
||
!output ||
|
||
!referenceRoot ||
|
||
dirs.length === 0 ||
|
||
output !== resolve(output)
|
||
)
|
||
throw new Error(
|
||
"用法：node eval/forward/stats.ts <全新绝对输出目录> <参考副本根目录> <检验目录>...",
|
||
);
|
||
mkdirSync(output);
|
||
const summaries = dirs.map((dir) => {
|
||
const studyDir = join(dir, "study");
|
||
const records = readLedger(studyDir);
|
||
const board = new Board(records);
|
||
const config = frozenConfig(records);
|
||
const accounts = forwardAccounts(records);
|
||
const batch = board.decision("batch", 1)!;
|
||
const finals = finalSubmissions(records);
|
||
const scores = accounts.candidates.map((c) => {
|
||
const p = board.protocol(c.candidate);
|
||
const merged = batch.output.candidates.find(
|
||
(b) => b.id === c.candidate,
|
||
)!.merged;
|
||
const secondPredictions = finals.flatMap((s) =>
|
||
s.content.kind === "secondPrediction" &&
|
||
s.content.target === c.candidate &&
|
||
board.stage(s.content.stageId).round === 1
|
||
? [
|
||
{
|
||
researcher: s.content.author,
|
||
q5: s.content.content.range.q5,
|
||
seq: s.seq,
|
||
},
|
||
]
|
||
: [],
|
||
);
|
||
const authorEstimates = p.proposals.map((id) => {
|
||
const proposal = finals.find(
|
||
(s) => s.content.kind === "proposal" && s.content.item === id,
|
||
)!;
|
||
const seal = ofKind(records, "seal").find(
|
||
(s) => s.content.proposal === proposal.seq,
|
||
)!;
|
||
return {
|
||
proposal: id,
|
||
author: proposal.content.author,
|
||
range: seal.content.range,
|
||
proposalSeq: proposal.seq,
|
||
sealSeq: seal.seq,
|
||
};
|
||
});
|
||
const runs = c.runs.map((r) => {
|
||
const result = r.results[0] ?? null;
|
||
const score = validScore(result?.content.judged ?? null);
|
||
return {
|
||
run: r.plan.content.run,
|
||
planSeq: r.plan.seq,
|
||
started: r.starts.length > 0,
|
||
result,
|
||
score,
|
||
band: scoreBand(score, merged),
|
||
measuredMinusQ5: score === null ? null : score - merged.q5,
|
||
};
|
||
});
|
||
return {
|
||
study: config.studyId,
|
||
candidate: c.candidate,
|
||
name: p.factors.map((f) => f.name).join("；"),
|
||
merged,
|
||
secondPredictions,
|
||
authorEstimates,
|
||
runs,
|
||
adopted: board.decision("adopt", 1)?.output.adopted === c.candidate,
|
||
verificationCount: c.verificationCount,
|
||
repairCount: c.repairCount,
|
||
started: c.started,
|
||
manualRequired:
|
||
"路线按冻结协议归类；完整高上限路线单列 measuredMinusQ5；忠实度及误判见独立审阅回复",
|
||
};
|
||
});
|
||
const lightChecks = ofKind(records, "callStarted")
|
||
.filter((c) =>
|
||
["implementation", "repair"].includes(
|
||
(c.content.materials as TaskMaterial).kind,
|
||
),
|
||
)
|
||
.map((c) => ({
|
||
call: c.content.call,
|
||
seq: c.seq,
|
||
...lightCheckUsage(
|
||
join(studyDir, "calls", c.content.call, "sessions"),
|
||
config.lightChecks.maxCount,
|
||
),
|
||
}));
|
||
let kill = null;
|
||
if (existsSync(join(dir, "kill.json"))) {
|
||
const evidence = JSON.parse(
|
||
readFileSync(join(dir, "kill.json"), "utf8"),
|
||
) as {
|
||
run: string;
|
||
candidate: string;
|
||
beforeKillSeq: number;
|
||
container: string;
|
||
};
|
||
const snapshots = ["before", "after-kill", "before-resume"].map(
|
||
(part) =>
|
||
JSON.parse(
|
||
readFileSync(join(dir, `container-${part}.json`), "utf8"),
|
||
) as ContainerSnapshot,
|
||
);
|
||
const lifecycleWindow = JSON.parse(
|
||
readFileSync(join(dir, "container-events-window.json"), "utf8"),
|
||
) as LifecycleWindow;
|
||
const lifecycleEvents = readFileSync(
|
||
join(dir, "container-events.jsonl"),
|
||
"utf8",
|
||
)
|
||
.split("\n")
|
||
.filter(Boolean)
|
||
.map((line) => JSON.parse(line) as ContainerEvent);
|
||
const lifecycleChecks = sameContainerThroughResult(
|
||
snapshots,
|
||
lifecycleEvents,
|
||
evidence.container,
|
||
);
|
||
const target = accounts.runs.find(
|
||
(r) => r.plan.content.run === evidence.run,
|
||
)!;
|
||
const candidatePlans = accounts.runs.filter(
|
||
(r) =>
|
||
r.plan.content.purpose === "forward" &&
|
||
r.plan.content.object === evidence.candidate,
|
||
);
|
||
const atKill = JSON.parse(
|
||
readFileSync(join(dir, "ledger-at-kill.json"), "utf8"),
|
||
) as LedgerRecord[];
|
||
const killSeq = atKill.at(-1)!.seq;
|
||
kill = {
|
||
evidence,
|
||
killSeq,
|
||
snapshots,
|
||
snapshotMeaning: "杀前、杀后、恢复启动前；没有把快照当成恢复完成的证据",
|
||
lifecycleWindow,
|
||
lifecycleChecks,
|
||
lifecycleEvents,
|
||
charged: target.starts.length,
|
||
resultSeq: target.results[0]?.seq ?? null,
|
||
checks: {
|
||
sameContainer:
|
||
Object.values(lifecycleChecks).every(Boolean) &&
|
||
lifecycleWindow.run === evidence.run &&
|
||
lifecycleWindow.resultSeq === target.results[0]?.seq &&
|
||
Date.parse(lifecycleWindow.since) < Date.parse(lifecycleWindow.until),
|
||
chargedOnceNoRerun:
|
||
target.starts.length === 1 && candidatePlans.length === 1,
|
||
resultAndContinued:
|
||
!ofKind(atKill, "runResult").some(
|
||
(r) => r.content.run === evidence.run,
|
||
) &&
|
||
target.results.length === 1 &&
|
||
target.results[0]!.seq > killSeq &&
|
||
accounts.checks.adoptedAndStopped,
|
||
},
|
||
};
|
||
}
|
||
return {
|
||
dir,
|
||
study: config.studyId,
|
||
scores,
|
||
accounts,
|
||
lightChecks,
|
||
kill,
|
||
pass:
|
||
Object.values(accounts.checks).every(Boolean) &&
|
||
lightChecks.every((c) => c.pass) &&
|
||
(kill === null || Object.values(kill.checks).every(Boolean)),
|
||
};
|
||
});
|
||
const references = [
|
||
"16-real-1",
|
||
"15-real-1",
|
||
"15-tight-2",
|
||
"14-real-1",
|
||
"12-real-2",
|
||
].flatMap((name) => referenceScores(join(referenceRoot, name, "board.sqlite")));
|
||
jsonFile(join(output, "stats.json"), summaries);
|
||
jsonFile(
|
||
join(output, "predictions-scores.json"),
|
||
summaries.flatMap((s) => s.scores),
|
||
);
|
||
jsonFile(join(output, "reference-scores.json"), references);
|
||
jsonFile(join(output, "score-comparison.json"), {
|
||
references,
|
||
ours: summaries.flatMap((s) => s.scores),
|
||
caveat:
|
||
"参考项目提示词带题目具体做法、方法版本也不同；保留无效成绩及重复尝试，不作公平因果比较",
|
||
});
|
||
jsonFile(join(output, "manual-review.json"), {
|
||
implementation:
|
||
"每个已启动候选一个全新上下文：协议逐条、基线、最终代码、concerns、位置表；逐项漏做/额外改进/位置表准确性和代码位置",
|
||
verification:
|
||
"同一审阅者再读各次核验及对应版本：误放行/误拒、修复是否仅改指出处；保存原始回复",
|
||
route:
|
||
"逐候选按任务书15附录一路线归类，并单列完整高上限路线的实测-q5；不自动按关键词贴标签",
|
||
comparison:
|
||
"复用eval/route导出、items、thinking、verify-ranges、原四问Jev及原阅读问卷，填A–J；与任务15三次历史结果并排，标历史批次",
|
||
serial:
|
||
"核对各目录launches.jsonl、machine.jsonl和父会话观察；上一份全部结束后才起下一份；不宣称快照排除任意外部进程",
|
||
});
|
||
console.log(
|
||
`统计：${output}；所给 ${summaries.length} 份研究的逐研究检查${summaries.every((s) => s.pass) ? "通过" : "未通过"}，不是任务书16总体验收；总体程序验收另运行 node eval/forward/acceptance.ts ${output}；质量判断仍待独立阅读。`,
|
||
);
|
||
if (!summaries.every((s) => s.pass)) process.exitCode = 1;
|