feat: 诊断真实运行与评估脚本（任务书 18 第三块）：起研究、杀诊断运行、统计、导出审阅材料
Co-Authored-By: Claude Sonnet 5.5 <noreply@anthropic.com>
This commit is contained in:
@@ -0,0 +1,310 @@
|
||||
import type { LedgerRecord } from "../../src/ledger/index.ts";
|
||||
import {
|
||||
Board,
|
||||
diagnosisId,
|
||||
type StudyRecord,
|
||||
} from "../../src/materials/index.ts";
|
||||
import {
|
||||
covers,
|
||||
improvement,
|
||||
placement,
|
||||
runBalance,
|
||||
} from "../../src/rules/index.ts";
|
||||
import { checkTaskFacts } from "../../src/tasks/index.ts";
|
||||
import { fullAccounts } from "./full.ts";
|
||||
import { frozenConfig } from "./forward.ts";
|
||||
import { ofKind } from "./ledger.ts";
|
||||
|
||||
// 诊断开、消融关的完整研究（任务书 18 第三块）的账目与逐轮事实：只从账本记录重算，不作质量判断。
|
||||
// 第 4 样"沿这个方向"这类判断由子 Agent 读材料标注，这里只给能由账本算出的列。
|
||||
|
||||
/** 消融关着时，每轮都应当有"按配置跳过"的七个步骤（src/config/steps.ts 的 off 清单）。 */
|
||||
export const ABLATION_STEPS = [
|
||||
"listCells",
|
||||
"explanation",
|
||||
"selectCells",
|
||||
"ablationImplementation",
|
||||
"ablationVerification",
|
||||
"ablationRun",
|
||||
"settle",
|
||||
] as const;
|
||||
|
||||
const BAND = {
|
||||
belowQ1: "<q1",
|
||||
q1ToQ5: "[q1,q5)",
|
||||
q5ToQ9: "[q5,q9]",
|
||||
aboveQ9: ">q9",
|
||||
} as const;
|
||||
|
||||
export function diagnosisAccounts(records: LedgerRecord[]) {
|
||||
const base = fullAccounts(records);
|
||||
const board = new Board(records);
|
||||
const config = frozenConfig(records);
|
||||
const facts = checkTaskFacts(
|
||||
(ofKind(records, "config")[0]!.content as unknown as StudyRecord).task,
|
||||
);
|
||||
const events = ofKind(records, "event");
|
||||
const opens = ofKind(records, "stageOpen");
|
||||
const closes = ofKind(records, "stageClose");
|
||||
const closeSeq = (stageId: string) =>
|
||||
closes.find((c) => c.content.stageId === stageId)?.seq ?? null;
|
||||
const planned = ofKind(records, "runPlanned");
|
||||
const started = ofKind(records, "runStarted");
|
||||
const results = ofKind(records, "runResult");
|
||||
|
||||
// 开过的轮次：有阶段、决定或按配置跳过事件的都算。
|
||||
const roundIds = base.rounds.map((r) => r.round);
|
||||
const rounds = roundIds.map((round) => {
|
||||
const roundOf = base.rounds.find((r) => r.round === round)!;
|
||||
const balanceBefore = runBalance(
|
||||
config.runs,
|
||||
board.runStatesBefore(round),
|
||||
).balance;
|
||||
const prefix = `r${round}-`;
|
||||
const diagStages = opens.filter(
|
||||
(o) =>
|
||||
o.content.round === round &&
|
||||
o.content.stageId.startsWith(`${prefix}diagnosis`),
|
||||
);
|
||||
const select = board.decision("selectDiagnosis", round) ?? null;
|
||||
const dedup = board.decision("diagnosisDedup", round) ?? null;
|
||||
const settlement = board.diagnosisSettlement(round) ?? null;
|
||||
const settlementRecords = ofKind(records, "diagnosisSettlement").filter(
|
||||
(r) => r.content.round === round,
|
||||
);
|
||||
const diagRuns = planned
|
||||
.filter(
|
||||
(p) => p.content.purpose === "diagnosis" && p.content.round === round,
|
||||
)
|
||||
.map((p) => ({
|
||||
run: p.content.run,
|
||||
planSeq: p.seq,
|
||||
startSeqs: started
|
||||
.filter((s) => s.content.run === p.content.run)
|
||||
.map((s) => s.seq),
|
||||
result: results.find((r) => r.content.run === p.content.run) ?? null,
|
||||
}));
|
||||
const anyDiagnosisRecord =
|
||||
diagStages.length > 0 ||
|
||||
select !== null ||
|
||||
dedup !== null ||
|
||||
diagRuns.length > 0 ||
|
||||
settlementRecords.length > 0;
|
||||
const merged = dedup === null ? [] : board.diagnosisExperiments(round);
|
||||
const dropped = settlement?.dropped ?? [];
|
||||
const verifications = board
|
||||
.published("diagnosisVerification")
|
||||
.filter((s) => s.round === round);
|
||||
const repairs = board
|
||||
.published("diagnosisRepair")
|
||||
.filter((s) => s.round === round);
|
||||
const predictionClose = closeSeq(`${prefix}diagnosisPrediction`);
|
||||
const firstStart = diagRuns.flatMap((r) => r.startSeqs).sort((a, b) => a - b)[0];
|
||||
const experiments = merged.map((e) => {
|
||||
const row = settlement?.output.experiments.find(
|
||||
(x) => x.experiment === e.id,
|
||||
);
|
||||
const judged = settlement?.judged.experiments.find((x) => x.id === e.id);
|
||||
const skip = select?.output.experiments.find(
|
||||
(x) => x.experiment === e.id,
|
||||
);
|
||||
return {
|
||||
id: e.id,
|
||||
name: e.name,
|
||||
spot: e.spot,
|
||||
authors: e.authors,
|
||||
chosen: select?.output.chosen.includes(e.id) ?? false,
|
||||
skipReason: skip?.reason ?? null,
|
||||
skipBestNet: skip?.bestCombo?.net ?? null,
|
||||
priorGain: select?.output.priorGains.find((g) => g.experiment === e.id)
|
||||
?.gain ?? null,
|
||||
outcome: row?.outcome ?? null,
|
||||
score:
|
||||
judged !== undefined &&
|
||||
judged.verdict === "valid" &&
|
||||
judged.score.status === "measured"
|
||||
? judged.score.value
|
||||
: null,
|
||||
scoreStatus: judged?.score.status ?? null,
|
||||
gain: row?.gain ?? null,
|
||||
merged: row?.merged ?? null,
|
||||
ending: judged?.ending ?? null,
|
||||
wallSeconds:
|
||||
judged?.wallSeconds.status === "measured"
|
||||
? judged.wallSeconds.value
|
||||
: null,
|
||||
worldBits: row?.worldBits ?? null,
|
||||
surprise:
|
||||
row?.surprise?.bits.status === "measured"
|
||||
? row.surprise.bits.value
|
||||
: null,
|
||||
dropped: dropped.find((d) => d.experiment === e.id)?.reason ?? null,
|
||||
};
|
||||
});
|
||||
|
||||
// 第 5 样：每位作者每条解释对每个有效实验的三个数，实测落点；合并范围的落点。
|
||||
const predictionRows =
|
||||
settlement === null
|
||||
? []
|
||||
: settlement.input.explanations.flatMap((x) =>
|
||||
settlement.input.experiments.flatMap((e) => {
|
||||
if (e.outcome.kind !== "valid") return [];
|
||||
const range = x.experiments.find(
|
||||
(y) => y.experiment === e.experiment,
|
||||
)!.range;
|
||||
return [
|
||||
{
|
||||
explanation: x.id,
|
||||
author: x.author,
|
||||
experiment: e.experiment,
|
||||
range,
|
||||
actual: e.outcome.score,
|
||||
band: BAND[placement(range, e.outcome.score)],
|
||||
covered: covers(range, e.outcome.score),
|
||||
},
|
||||
];
|
||||
}),
|
||||
);
|
||||
const mergedRows =
|
||||
settlement === null
|
||||
? []
|
||||
: settlement.input.experiments.flatMap((e) => {
|
||||
if (e.outcome.kind !== "valid") return [];
|
||||
const range = settlement.output.experiments.find(
|
||||
(x) => x.experiment === e.experiment,
|
||||
)!.merged;
|
||||
return [
|
||||
{
|
||||
experiment: e.experiment,
|
||||
range,
|
||||
actual: e.outcome.score,
|
||||
band: BAND[placement(range, e.outcome.score)],
|
||||
covered: covers(range, e.outcome.score),
|
||||
},
|
||||
];
|
||||
});
|
||||
const medians = (
|
||||
settlement?.output.experiments.map((x) => x.merged.q5) ?? []
|
||||
).sort((a, b) => a - b);
|
||||
const experimentGap =
|
||||
medians.length < 2
|
||||
? null
|
||||
: {
|
||||
maxMinusMin: medians.at(-1)! - medians[0]!,
|
||||
minPairwise: Math.min(
|
||||
...medians.slice(1).map((m, i) => m - medians[i]!),
|
||||
),
|
||||
};
|
||||
const batch = board.decision("batch", round) ?? null;
|
||||
const candMedians = (batch?.output.candidates ?? [])
|
||||
.flatMap((c) => (c.merged === null ? [] : [c.merged.q5]))
|
||||
.sort((a, b) => a - b);
|
||||
const candidateGap =
|
||||
candMedians.length < 2
|
||||
? null
|
||||
: {
|
||||
maxMinusMin: candMedians.at(-1)! - candMedians[0]!,
|
||||
minPairwise: Math.min(
|
||||
...candMedians.slice(1).map((m, i) => m - candMedians[i]!),
|
||||
),
|
||||
};
|
||||
|
||||
// 本轮用于第 4 样的正向侧：每个候选相对本轮基线的改进量。
|
||||
const baseline = board.baselineOf(round);
|
||||
const baselineScore = board.baselineScore(baseline);
|
||||
const forward = roundOf.candidates.map((c) => ({
|
||||
candidate: c.candidate,
|
||||
score: c.score,
|
||||
gain:
|
||||
c.score === null
|
||||
? null
|
||||
: improvement(baselineScore, c.score, facts.finalMetric.direction),
|
||||
}));
|
||||
|
||||
// 本轮诊断结算按轮次唯一；运行扣次：确认启动的诊断运行数。
|
||||
const ablationAbsent =
|
||||
opens.every(
|
||||
(o) =>
|
||||
o.content.round !== round ||
|
||||
!["explanation", "ablationImplementation", "ablationVerification"].includes(
|
||||
o.content.stage,
|
||||
),
|
||||
) &&
|
||||
planned.every(
|
||||
(p) => p.content.purpose !== "ablation" || p.content.round !== round,
|
||||
) &&
|
||||
board.decision("cells", round) === undefined &&
|
||||
board.decision("bet", round) === undefined &&
|
||||
board.settlement(round) === undefined;
|
||||
const skippedSteps = ABLATION_STEPS.map((step) =>
|
||||
events.some(
|
||||
(e) =>
|
||||
e.content.event === "skippedByConfig" &&
|
||||
e.content.objects[0] === `${prefix}${step}`,
|
||||
),
|
||||
);
|
||||
return {
|
||||
round,
|
||||
baseline,
|
||||
baselineScore,
|
||||
balanceBefore,
|
||||
expectedDiagnosis: balanceBefore >= 2,
|
||||
anyDiagnosisRecord,
|
||||
diagnosed: select?.output.diagnose ?? false,
|
||||
chosen: select?.output.chosen ?? [],
|
||||
chosenNet: select?.output.chosenNet ?? null,
|
||||
combos: select?.output.combos ?? [],
|
||||
g0: select?.output.g0 ?? null,
|
||||
selectParams: select?.output.params ?? null,
|
||||
experiments,
|
||||
verificationCount: verifications.length,
|
||||
repairCount: repairs.length,
|
||||
droppedCount: dropped.length,
|
||||
ranCount: settlement?.judged.experiments.length ?? 0,
|
||||
runs: diagRuns,
|
||||
charged: diagRuns.filter((r) => r.startSeqs.length > 0).length,
|
||||
settlementCount: settlementRecords.length,
|
||||
direction: settlement?.output.direction ?? null,
|
||||
judgments: settlement?.output.judgments ?? null,
|
||||
predictionRows,
|
||||
mergedRows,
|
||||
experimentGap,
|
||||
candidateGap,
|
||||
forward,
|
||||
predictionCloseSeq: predictionClose,
|
||||
firstDiagnosisStartSeq: firstStart ?? null,
|
||||
predictionBeforeStart:
|
||||
firstStart === undefined
|
||||
? null
|
||||
: predictionClose !== null && predictionClose < firstStart,
|
||||
ablationAbsent,
|
||||
ablationSkipEventsAll: skippedSteps.every(Boolean),
|
||||
diagnosisNotRunEvents: events.filter(
|
||||
(e) =>
|
||||
e.content.event === "diagnosisNotRun" &&
|
||||
e.content.objects.includes(diagnosisId(round)),
|
||||
).length,
|
||||
};
|
||||
});
|
||||
|
||||
const checks = {
|
||||
...base.checks,
|
||||
diagnosisChargedEqualsStarts: rounds.every(
|
||||
(r) => r.runs.length === 0 || r.charged === r.runs.filter((x) => x.startSeqs.length > 0).length,
|
||||
),
|
||||
oneDiagnosisSettlementPerRound: rounds.every((r) => r.settlementCount <= 1),
|
||||
noDiagnosisBelowTwo: rounds.every(
|
||||
(r) => r.expectedDiagnosis || !r.anyDiagnosisRecord,
|
||||
),
|
||||
diagnosisWhenBalanceAtLeastTwo: rounds.every(
|
||||
(r) => !r.expectedDiagnosis || r.anyDiagnosisRecord,
|
||||
),
|
||||
predictionBeforeStart: rounds.every((r) => r.predictionBeforeStart !== false),
|
||||
ablationOffClean: rounds.every((r) => r.ablationAbsent),
|
||||
ablationSkipEvents: rounds.every((r) => r.ablationSkipEventsAll),
|
||||
diagnosisRunPurposeCount:
|
||||
planned.filter((p) => p.content.purpose === "diagnosis").length ===
|
||||
rounds.reduce((n, r) => n + r.runs.length, 0),
|
||||
};
|
||||
return { ...base, diagnosisRounds: rounds, diagnosisChecks: checks };
|
||||
}
|
||||
Reference in New Issue
Block a user