eval: 准备正向采纳的真实运行恢复与质量评估
新增单份串行启动入口、首个正向运行杀进程恢复证据、逐候选审阅导出与账目统计；参考只读副本分数对照。456项免费测试通过，尚未启动三份真实验收。同步结构说明与当前工作状态。
This commit is contained in:
@@ -0,0 +1,132 @@
|
||||
import { mkdirSync, writeFileSync } from "node:fs";
|
||||
import { join, resolve } from "node:path";
|
||||
import {
|
||||
Board,
|
||||
protocolMetrics,
|
||||
type TaskMaterial,
|
||||
} from "../../src/materials/index.ts";
|
||||
import { checkoutVersion, compareVersions } from "../../src/workspace/index.ts";
|
||||
import { candidateHistory, jsonFile } from "../lib/forward.ts";
|
||||
import { ofKind, readLedger } from "../lib/ledger.ts";
|
||||
|
||||
// 任务书16五项质量评估的候选审阅材料；不判路线、不代替全新上下文逐条阅读。
|
||||
// node eval/forward/export.ts <研究目录> <全新绝对输出目录>
|
||||
const [study, output, ...extra] = process.argv.slice(2);
|
||||
if (!study || !output || extra.length || output !== resolve(output))
|
||||
throw new Error(
|
||||
"用法：node eval/forward/export.ts <研究目录> <全新绝对输出目录>",
|
||||
);
|
||||
mkdirSync(output); // 不覆盖以前的审阅材料。
|
||||
const records = readLedger(study);
|
||||
const board = new Board(records);
|
||||
const repo = join(study, "repo.git");
|
||||
const batch = board.decision("batch", 1)!;
|
||||
const baseline = board.code(board.baselineOf(1));
|
||||
const plans = ofKind(records, "runPlanned").filter(
|
||||
(p) => p.content.round === 1 && p.content.purpose === "forward",
|
||||
);
|
||||
const index = [];
|
||||
for (const candidate of batch.output.chosen) {
|
||||
const dir = join(output, candidate);
|
||||
mkdirSync(dir);
|
||||
const history = candidateHistory(records, candidate);
|
||||
const candidatePlans = plans.filter((p) => p.content.object === candidate);
|
||||
const started = candidatePlans.filter((p) =>
|
||||
ofKind(records, "runStarted").some((s) => s.content.run === p.content.run),
|
||||
);
|
||||
const protocol = ofKind(records, "protocol")
|
||||
.filter((p) => p.content.candidate === candidate)
|
||||
.at(-1)!;
|
||||
jsonFile(join(dir, "protocol.json"), {
|
||||
record: protocol,
|
||||
resolvedMetrics: protocolMetrics(board, board.protocol(candidate)),
|
||||
});
|
||||
mkdirSync(join(dir, "baseline"));
|
||||
checkoutVersion(repo, baseline.version, join(dir, "baseline"));
|
||||
for (const attempt of history.attempts) {
|
||||
const path = join(dir, `code-${attempt.code.seq}`);
|
||||
mkdirSync(path);
|
||||
checkoutVersion(repo, attempt.code.content.version, path);
|
||||
jsonFile(
|
||||
join(dir, `changes-${attempt.code.seq}.json`),
|
||||
compareVersions(repo, baseline.version, attempt.code.content.version),
|
||||
);
|
||||
}
|
||||
// 首轮里每个候选只正式跑一个已通过版本；无启动的候选仍导出历史，但不冒称有最终通过代码。
|
||||
const firstRun = started[0];
|
||||
const finalCode =
|
||||
firstRun === undefined
|
||||
? null
|
||||
: ofKind(records, "code")
|
||||
.filter((c) => c.content.object === candidate && c.seq < firstRun.seq)
|
||||
.at(-1)!;
|
||||
if (finalCode !== null) {
|
||||
mkdirSync(join(dir, "final"));
|
||||
checkoutVersion(repo, finalCode.content.version, join(dir, "final"));
|
||||
}
|
||||
const finalAttempt =
|
||||
finalCode === null
|
||||
? null
|
||||
: history.attempts.find((a) => a.code.seq === finalCode.seq)!;
|
||||
jsonFile(join(dir, "final-implementation.json"), finalAttempt);
|
||||
jsonFile(join(dir, "attempts.json"), history.attempts);
|
||||
jsonFile(join(dir, "verifications.json"), history.verifications);
|
||||
const calls = ofKind(records, "callStarted").filter((c) => {
|
||||
const material = c.content.materials as TaskMaterial;
|
||||
return (
|
||||
material.object === candidate &&
|
||||
["implementation", "repair", "verification"].includes(material.kind)
|
||||
);
|
||||
});
|
||||
jsonFile(
|
||||
join(dir, "calls.json"),
|
||||
calls.map((c) => ({
|
||||
record: c,
|
||||
sessions: join(study, "calls", c.content.call, "sessions"),
|
||||
})),
|
||||
);
|
||||
jsonFile(
|
||||
join(dir, "rejections.json"),
|
||||
ofKind(records, "rejection").filter(
|
||||
(r) =>
|
||||
r.content.target === candidate &&
|
||||
["implementation", "repair", "verification"].includes(r.content.kind),
|
||||
),
|
||||
);
|
||||
const manifest = {
|
||||
study,
|
||||
candidate,
|
||||
protocolSeq: protocol.seq,
|
||||
baselineVersion: baseline.version,
|
||||
finalCode,
|
||||
ran: started.length > 0,
|
||||
runs: candidatePlans.map((p) => ({
|
||||
plan: p,
|
||||
result:
|
||||
ofKind(records, "runResult").find(
|
||||
(r) => r.content.run === p.content.run,
|
||||
) ?? null,
|
||||
artifacts: join(study, "runs", p.content.run),
|
||||
})),
|
||||
};
|
||||
jsonFile(join(dir, "manifest.json"), manifest);
|
||||
writeFileSync(
|
||||
join(dir, "review.txt"),
|
||||
`任务书16第3.2节：只读本候选，分两步评估，中文直接回复，不写文件。\n` +
|
||||
`第一步只读 protocol.json（冻结协议全文及引用指标原定义）、baseline/、final/、final-implementation.json（正文、位置表、concerns）。\n` +
|
||||
`逐条实现条款、因子、保障检查漏做、协议之外的改进、位置表是否准确，每条附代码文件和行号。先形成独立结论，再打开核验历史。\n` +
|
||||
`第二步读 verifications.json、attempts.json、code-<序号>/、changes-<序号>.json；每次核验对照其中code指向的那一版，不拿最终修好版本倒判历史。判断该拒却通过、不该拒却拒绝；修复是否只改指出处。\n` +
|
||||
`需要时读 calls.json 和 rejections.json；模型材料和原始会话都可追溯。不把提交格式退回当成核验失败。\n` +
|
||||
`答案路径只核对每个可能交出最终答案分支的标签，不预测实际走哪个分支。报告先写实现忠实度，再写逐次核验误判和修复范围。\n` +
|
||||
`按任务书15附录一路线代号归类冻结协议，另标是否完整高上限路线（起点、连续改进全部圆、多起点或扰动、合法收尾），附协议原文位置。\n` +
|
||||
`本候选${manifest.ran ? "已启动正式运行" : "未启动正式运行；没有final/，只评历史，不列成跑过的候选"}。\n`,
|
||||
);
|
||||
index.push({
|
||||
candidate,
|
||||
ran: manifest.ran,
|
||||
directory: dir,
|
||||
finalVersion: finalCode?.content.version ?? null,
|
||||
});
|
||||
}
|
||||
jsonFile(join(output, "index.json"), index);
|
||||
console.log(`候选审阅包：${output}`);
|
||||
@@ -0,0 +1,259 @@
|
||||
import { execFileSync } from "node:child_process";
|
||||
import { appendFileSync, existsSync, mkdirSync } from "node:fs";
|
||||
import { join, resolve } from "node:path";
|
||||
import { forwardAccounts, frozenConfig, jsonFile } from "../lib/forward.ts";
|
||||
import { ofKind, readLedger } from "../lib/ledger.ts";
|
||||
import { launch, sleep, waitForLedger, writeConfig } from "../lib/study.ts";
|
||||
import { watchTimeline } from "../lib/timeline.ts";
|
||||
|
||||
// 任务书16：一次只启动一份；三次由父会话逐个执行。kill 只打首个正向运行，随后恢复同一研究一次。
|
||||
// node eval/forward/run.ts <new|kill|resume> <研究编号> <绝对输出目录>
|
||||
const [mode, name, output, ...extra] = process.argv.slice(2);
|
||||
if (
|
||||
!mode ||
|
||||
!name ||
|
||||
!output ||
|
||||
extra.length ||
|
||||
!["new", "kill", "resume"].includes(mode) ||
|
||||
output !== resolve(output)
|
||||
) {
|
||||
throw new Error(
|
||||
"用法：node eval/forward/run.ts <new|kill|resume> <研究编号> <绝对输出目录>",
|
||||
);
|
||||
}
|
||||
const out = output;
|
||||
const studyDir = join(out, "study");
|
||||
if (mode === "resume") {
|
||||
if (frozenConfig(readLedger(studyDir)).studyId !== name)
|
||||
throw new Error("研究编号与账本不符");
|
||||
} else {
|
||||
if (existsSync(out))
|
||||
throw new Error(`输出目录已存在，不覆盖现场：${out}`);
|
||||
mkdirSync(out, { recursive: true });
|
||||
}
|
||||
const config =
|
||||
mode === "resume"
|
||||
? null
|
||||
: writeConfig(
|
||||
"eval/round-zero/research.json",
|
||||
join(out, "research.json"),
|
||||
name,
|
||||
null,
|
||||
);
|
||||
const inspect = (container: string) =>
|
||||
JSON.parse(
|
||||
execFileSync("docker", ["inspect", container], {
|
||||
encoding: "utf8",
|
||||
}),
|
||||
)[0] as {
|
||||
Id: string;
|
||||
Name: string;
|
||||
Created: string;
|
||||
State: {
|
||||
Running: boolean;
|
||||
StartedAt: string;
|
||||
FinishedAt: string;
|
||||
Status: string;
|
||||
};
|
||||
};
|
||||
function observe(label: string) {
|
||||
appendFileSync(
|
||||
join(out, "machine.jsonl"),
|
||||
`${JSON.stringify({
|
||||
at: new Date().toISOString(),
|
||||
label,
|
||||
processes: execFileSync(
|
||||
"ps",
|
||||
["-eo", "pid,ppid,lstart,args"],
|
||||
{ encoding: "utf8" },
|
||||
),
|
||||
containers: execFileSync(
|
||||
"docker",
|
||||
["ps", "--no-trunc"],
|
||||
{ encoding: "utf8" },
|
||||
),
|
||||
})}\n`,
|
||||
);
|
||||
}
|
||||
function start(path: string | null) {
|
||||
observe(path === null ? "resume" : "start");
|
||||
const study = launch(path, studyDir, {});
|
||||
appendFileSync(
|
||||
join(out, "launches.jsonl"),
|
||||
`${JSON.stringify({ event: "start", pid: study.child.pid, at: new Date().toISOString(), config: path })}\n`,
|
||||
);
|
||||
const interrupt = () => study.child.kill("SIGINT");
|
||||
process.on("SIGINT", interrupt);
|
||||
study.done.then((end) => {
|
||||
process.off("SIGINT", interrupt);
|
||||
appendFileSync(
|
||||
join(out, "launches.jsonl"),
|
||||
`${JSON.stringify({ event: "end", pid: study.child.pid, at: new Date().toISOString(), ...end })}\n`,
|
||||
);
|
||||
});
|
||||
return study;
|
||||
}
|
||||
let active = start(config);
|
||||
try {
|
||||
await waitForLedger(studyDir, active.child);
|
||||
let watching = watchTimeline(
|
||||
studyDir,
|
||||
active.child,
|
||||
join(out, "timeline.jsonl"),
|
||||
);
|
||||
if (mode === "kill") {
|
||||
let hit = false;
|
||||
while (
|
||||
active.child.exitCode === null &&
|
||||
active.child.signalCode === null
|
||||
) {
|
||||
const records = readLedger(studyDir);
|
||||
const plan = ofKind(records, "runPlanned").find(
|
||||
(r) =>
|
||||
r.content.purpose === "forward" &&
|
||||
r.content.round === 1,
|
||||
);
|
||||
if (plan !== undefined) {
|
||||
const run = plan.content.run;
|
||||
if (
|
||||
ofKind(records, "runResult").some(
|
||||
(r) => r.content.run === run,
|
||||
)
|
||||
)
|
||||
throw new Error(
|
||||
"首个正向运行已经结束，未打中；保留现场，不改杀别的候选",
|
||||
);
|
||||
const started = ofKind(
|
||||
records,
|
||||
"runStarted",
|
||||
).find((r) => r.content.run === run);
|
||||
if (started !== undefined) {
|
||||
// 容器名是正式入口记录在结构文档里的约定，不 import flow 内部文件。
|
||||
const container = `inforsi-${name}-${run}`;
|
||||
const before = inspect(container);
|
||||
jsonFile(
|
||||
join(
|
||||
out,
|
||||
"container-before.json",
|
||||
),
|
||||
before,
|
||||
);
|
||||
if (!before.State.Running)
|
||||
throw new Error(
|
||||
"首个正向容器已不在运行，未打中",
|
||||
);
|
||||
if (
|
||||
ofKind(
|
||||
readLedger(studyDir),
|
||||
"runResult",
|
||||
).some(
|
||||
(r) =>
|
||||
r.content
|
||||
.run ===
|
||||
run,
|
||||
)
|
||||
)
|
||||
throw new Error(
|
||||
"杀前已有结果，未打中",
|
||||
);
|
||||
jsonFile(join(out, "kill.json"), {
|
||||
run,
|
||||
candidate: plan.content.object,
|
||||
planSeq: plan.seq,
|
||||
startSeq: started.seq,
|
||||
beforeKillSeq:
|
||||
records.at(-1)!.seq,
|
||||
pid: active.child.pid,
|
||||
at: new Date().toISOString(),
|
||||
container,
|
||||
});
|
||||
if (!active.child.kill("SIGKILL"))
|
||||
throw new Error(
|
||||
"未发出 SIGKILL",
|
||||
);
|
||||
const killed = await active.done;
|
||||
await watching;
|
||||
if (killed.signal !== "SIGKILL")
|
||||
throw new Error(
|
||||
"控制进程不是以 SIGKILL 结束",
|
||||
);
|
||||
const atKill = readLedger(studyDir);
|
||||
jsonFile(
|
||||
join(
|
||||
out,
|
||||
"ledger-at-kill.json",
|
||||
),
|
||||
atKill,
|
||||
);
|
||||
if (
|
||||
ofKind(
|
||||
atKill,
|
||||
"runResult",
|
||||
).some(
|
||||
(r) =>
|
||||
r.content
|
||||
.run ===
|
||||
run,
|
||||
)
|
||||
)
|
||||
throw new Error(
|
||||
"杀后核对发现目标运行已经写结果，未打中；保留现场",
|
||||
);
|
||||
const afterKill = inspect(container);
|
||||
jsonFile(
|
||||
join(
|
||||
out,
|
||||
"container-after-kill.json",
|
||||
),
|
||||
afterKill,
|
||||
);
|
||||
observe("after-kill");
|
||||
active = start(null);
|
||||
const afterResume = inspect(container);
|
||||
jsonFile(
|
||||
join(
|
||||
out,
|
||||
"container-after-resume.json",
|
||||
),
|
||||
afterResume,
|
||||
);
|
||||
watching = watchTimeline(
|
||||
studyDir,
|
||||
active.child,
|
||||
join(out, "timeline.jsonl"),
|
||||
);
|
||||
hit = true;
|
||||
break;
|
||||
}
|
||||
}
|
||||
await sleep(100);
|
||||
}
|
||||
if (!hit)
|
||||
throw new Error(
|
||||
`没有打中首个正向运行：${JSON.stringify(await active.done)}`,
|
||||
);
|
||||
}
|
||||
await watching;
|
||||
const end = await active.done;
|
||||
jsonFile(join(out, "outcome.json"), end);
|
||||
observe("end");
|
||||
const accounts = forwardAccounts(readLedger(studyDir));
|
||||
jsonFile(join(out, "accounts.json"), accounts);
|
||||
// 杀进程报告由 stats 读原始容器观察和账本重算；即使外部服务失败后人工恢复，也不覆盖原杀进程证据。
|
||||
if (end.code !== 0 || !Object.values(accounts.checks).every(Boolean)) {
|
||||
throw new Error(
|
||||
`没有完成采纳验收，保留现场；查明原因后才能手动 resume：${JSON.stringify(end)}`,
|
||||
);
|
||||
}
|
||||
console.log(
|
||||
`已到采纳后停点：${studyDir}。仍需 export、stats 和独立质量审阅。`,
|
||||
);
|
||||
} finally {
|
||||
if (
|
||||
active.child.exitCode === null &&
|
||||
active.child.signalCode === null
|
||||
) {
|
||||
active.child.kill("SIGINT");
|
||||
await active.done;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,216 @@
|
||||
import { existsSync, mkdirSync, readFileSync } from "node:fs";
|
||||
import { join, resolve } from "node:path";
|
||||
import type { LedgerRecord } from "../../src/ledger/index.ts";
|
||||
import {
|
||||
Board,
|
||||
validScore,
|
||||
type TaskMaterial,
|
||||
} from "../../src/materials/index.ts";
|
||||
import {
|
||||
forwardAccounts,
|
||||
frozenConfig,
|
||||
jsonFile,
|
||||
scoreBand,
|
||||
} from "../lib/forward.ts";
|
||||
import { finalSubmissions, ofKind, readLedger } from "../lib/ledger.ts";
|
||||
import { lightCheckUsage } from "../lib/light-checks.ts";
|
||||
import { referenceScores } from "../lib/reference-scores.ts";
|
||||
|
||||
// 五项表中可计算的字段。路线与审阅结论故意不自动猜，由父会话按候选审阅包填写报告。
|
||||
// node eval/forward/stats.ts <全新绝对输出目录> <五份参考副本根目录> <检验目录>...
|
||||
const [output, referenceRoot, ...dirs] = process.argv.slice(2);
|
||||
if (
|
||||
!output ||
|
||||
!referenceRoot ||
|
||||
dirs.length === 0 ||
|
||||
output !== resolve(output)
|
||||
)
|
||||
throw new Error(
|
||||
"用法：node eval/forward/stats.ts <全新绝对输出目录> <参考副本根目录> <检验目录>...",
|
||||
);
|
||||
mkdirSync(output);
|
||||
const summaries = dirs.map((dir) => {
|
||||
const studyDir = join(dir, "study");
|
||||
const records = readLedger(studyDir);
|
||||
const board = new Board(records);
|
||||
const config = frozenConfig(records);
|
||||
const accounts = forwardAccounts(records);
|
||||
const batch = board.decision("batch", 1)!;
|
||||
const finals = finalSubmissions(records);
|
||||
const scores = accounts.candidates.map((c) => {
|
||||
const p = board.protocol(c.candidate);
|
||||
const merged = batch.output.candidates.find(
|
||||
(b) => b.id === c.candidate,
|
||||
)!.merged;
|
||||
const secondPredictions = finals.flatMap((s) =>
|
||||
s.content.kind === "secondPrediction" &&
|
||||
s.content.target === c.candidate &&
|
||||
board.stage(s.content.stageId).round === 1
|
||||
? [
|
||||
{
|
||||
researcher: s.content.author,
|
||||
q5: s.content.content.range.q5,
|
||||
seq: s.seq,
|
||||
},
|
||||
]
|
||||
: [],
|
||||
);
|
||||
const authorEstimates = p.proposals.map((id) => {
|
||||
const proposal = finals.find(
|
||||
(s) => s.content.kind === "proposal" && s.content.item === id,
|
||||
)!;
|
||||
const seal = ofKind(records, "seal").find(
|
||||
(s) => s.content.proposal === proposal.seq,
|
||||
)!;
|
||||
return {
|
||||
proposal: id,
|
||||
author: proposal.content.author,
|
||||
range: seal.content.range,
|
||||
proposalSeq: proposal.seq,
|
||||
sealSeq: seal.seq,
|
||||
};
|
||||
});
|
||||
const runs = c.runs.map((r) => {
|
||||
const result = r.results[0] ?? null;
|
||||
const score = validScore(result?.content.judged ?? null);
|
||||
return {
|
||||
run: r.plan.content.run,
|
||||
planSeq: r.plan.seq,
|
||||
started: r.starts.length > 0,
|
||||
result,
|
||||
score,
|
||||
band: scoreBand(score, merged),
|
||||
measuredMinusQ5: score === null ? null : score - merged.q5,
|
||||
};
|
||||
});
|
||||
return {
|
||||
study: config.studyId,
|
||||
candidate: c.candidate,
|
||||
name: p.factors.map((f) => f.name).join("；"),
|
||||
merged,
|
||||
secondPredictions,
|
||||
authorEstimates,
|
||||
runs,
|
||||
adopted: board.decision("adopt", 1)?.output.adopted === c.candidate,
|
||||
verificationCount: c.verificationCount,
|
||||
repairCount: c.repairCount,
|
||||
started: c.started,
|
||||
manualRequired:
|
||||
"路线按冻结协议归类；完整高上限路线单列 measuredMinusQ5；忠实度及误判见独立审阅回复",
|
||||
};
|
||||
});
|
||||
const lightChecks = ofKind(records, "callStarted")
|
||||
.filter((c) =>
|
||||
["implementation", "repair"].includes(
|
||||
(c.content.materials as TaskMaterial).kind,
|
||||
),
|
||||
)
|
||||
.map((c) => ({
|
||||
call: c.content.call,
|
||||
seq: c.seq,
|
||||
...lightCheckUsage(
|
||||
join(studyDir, "calls", c.content.call, "sessions"),
|
||||
config.lightChecks.maxCount,
|
||||
),
|
||||
}));
|
||||
let kill = null;
|
||||
if (existsSync(join(dir, "kill.json"))) {
|
||||
const evidence = JSON.parse(
|
||||
readFileSync(join(dir, "kill.json"), "utf8"),
|
||||
) as { run: string; candidate: string; beforeKillSeq: number };
|
||||
type Container = {
|
||||
Id: string;
|
||||
State: { StartedAt: string; Running: boolean };
|
||||
};
|
||||
const snapshots = ["before", "after-kill", "after-resume"].map(
|
||||
(part) =>
|
||||
JSON.parse(
|
||||
readFileSync(join(dir, `container-${part}.json`), "utf8"),
|
||||
) as Container,
|
||||
);
|
||||
const target = accounts.runs.find(
|
||||
(r) => r.plan.content.run === evidence.run,
|
||||
)!;
|
||||
const candidatePlans = accounts.runs.filter(
|
||||
(r) =>
|
||||
r.plan.content.purpose === "forward" &&
|
||||
r.plan.content.object === evidence.candidate,
|
||||
);
|
||||
const atKill = JSON.parse(
|
||||
readFileSync(join(dir, "ledger-at-kill.json"), "utf8"),
|
||||
) as LedgerRecord[];
|
||||
const killSeq = atKill.at(-1)!.seq;
|
||||
kill = {
|
||||
evidence,
|
||||
killSeq,
|
||||
snapshots,
|
||||
charged: target.starts.length,
|
||||
resultSeq: target.results[0]?.seq ?? null,
|
||||
checks: {
|
||||
sameContainer:
|
||||
snapshots[0]!.State.Running &&
|
||||
snapshots.every(
|
||||
(c) =>
|
||||
c.Id === snapshots[0]!.Id &&
|
||||
c.State.StartedAt === snapshots[0]!.State.StartedAt,
|
||||
),
|
||||
chargedOnceNoRerun:
|
||||
target.starts.length === 1 && candidatePlans.length === 1,
|
||||
resultAndContinued:
|
||||
!ofKind(atKill, "runResult").some(
|
||||
(r) => r.content.run === evidence.run,
|
||||
) &&
|
||||
target.results.length === 1 &&
|
||||
target.results[0]!.seq > killSeq &&
|
||||
accounts.checks.adoptedAndStopped,
|
||||
},
|
||||
};
|
||||
}
|
||||
return {
|
||||
dir,
|
||||
study: config.studyId,
|
||||
scores,
|
||||
accounts,
|
||||
lightChecks,
|
||||
kill,
|
||||
pass:
|
||||
Object.values(accounts.checks).every(Boolean) &&
|
||||
lightChecks.every((c) => c.pass) &&
|
||||
(kill === null || Object.values(kill.checks).every(Boolean)),
|
||||
};
|
||||
});
|
||||
const references = [
|
||||
"16-real-1",
|
||||
"15-real-1",
|
||||
"15-tight-2",
|
||||
"14-real-1",
|
||||
"12-real-2",
|
||||
].flatMap((name) => referenceScores(join(referenceRoot, name, "board.sqlite")));
|
||||
jsonFile(join(output, "stats.json"), summaries);
|
||||
jsonFile(
|
||||
join(output, "predictions-scores.json"),
|
||||
summaries.flatMap((s) => s.scores),
|
||||
);
|
||||
jsonFile(join(output, "reference-scores.json"), references);
|
||||
jsonFile(join(output, "score-comparison.json"), {
|
||||
references,
|
||||
ours: summaries.flatMap((s) => s.scores),
|
||||
caveat:
|
||||
"参考项目提示词带题目具体做法、方法版本也不同；保留无效成绩及重复尝试，不作公平因果比较",
|
||||
});
|
||||
jsonFile(join(output, "manual-review.json"), {
|
||||
implementation:
|
||||
"每个已启动候选一个全新上下文：协议逐条、基线、最终代码、concerns、位置表；逐项漏做/额外改进/位置表准确性和代码位置",
|
||||
verification:
|
||||
"同一审阅者再读各次核验及对应版本：误放行/误拒、修复是否仅改指出处；保存原始回复",
|
||||
route:
|
||||
"逐候选按任务书15附录一路线归类，并单列完整高上限路线的实测-q5；不自动按关键词贴标签",
|
||||
comparison:
|
||||
"复用eval/route导出、items、thinking、verify-ranges、原四问Jev及原阅读问卷，填A–J；与任务15三次历史结果并排，标历史批次",
|
||||
serial:
|
||||
"核对各目录launches.jsonl、machine.jsonl和父会话观察；上一份全部结束后才起下一份；不宣称快照排除任意外部进程",
|
||||
});
|
||||
console.log(
|
||||
`统计：${output}；程序检查${summaries.every((s) => s.pass) ? "通过" : "未通过"}；质量判断仍待独立阅读。`,
|
||||
);
|
||||
if (!summaries.every((s) => s.pass)) process.exitCode = 1;
|
||||
Reference in New Issue
Block a user