From de8e13b0313271e43ea5ad131b983f5da291f430 Mon Sep 17 00:00:00 2001 From: iomgaa Date: Sun, 4 Oct 2026 04:52:13 -0400 Subject: [PATCH] =?UTF-8?q?fix:=20=E7=94=A8=E5=AE=B9=E5=99=A8=E7=94=9F?= =?UTF-8?q?=E5=91=BD=E5=91=A8=E6=9C=9F=E6=A0=B8=E5=AE=9E=E6=81=A2=E5=A4=8D?= =?UTF-8?q?=E5=B9=B6=E5=8D=95=E5=88=97=E4=B8=89=E4=BB=BD=E6=80=BB=E4=BD=93?= =?UTF-8?q?=E9=AA=8C=E6=94=B6?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 审阅发现spawn后快照早于恢复实际执行，不能证明没有重启；改用杀前至目标结果落账的Docker历史事件，保留缓存完整性限制。逐份stats不再冒称任务书总体通过；独立验收要求三份不同研究、成功杀进程及不重叠区间。两项缺陷附回归，7项定向测试与check通过，独立复审无新问题；真实B尚待启动。 --- eval/forward/acceptance.ts | 22 ++++ eval/forward/run.ts | 91 ++++++++++++++- eval/forward/stats.ts | 50 ++++++--- eval/lib/forward-evidence.ts | 125 +++++++++++++++++++++ tests/forward-eval.test.ts | 210 +++++++++++++++++++++++++++++++++-- 5 files changed, 473 insertions(+), 25 deletions(-) create mode 100644 eval/forward/acceptance.ts create mode 100644 eval/lib/forward-evidence.ts diff --git a/eval/forward/acceptance.ts b/eval/forward/acceptance.ts new file mode 100644 index 0000000..b3a773d --- /dev/null +++ b/eval/forward/acceptance.ts @@ -0,0 +1,22 @@ +import { readFileSync } from "node:fs"; +import { join } from "node:path"; +import { jsonFile } from "../lib/forward.ts"; +import { + forwardAcceptance, + type StudySummary, +} from "../lib/forward-evidence.ts"; + +// 显式总体程序验收：先对三份研究运行 stats，再用该统计目录检查任务书16的完整批次。 +// node eval/forward/acceptance.ts <统计输出目录> +const [statsDir, ...extra] = process.argv.slice(2); +if (!statsDir || extra.length) + throw new Error("用法：node eval/forward/acceptance.ts <统计输出目录>"); +const summaries = JSON.parse( + readFileSync(join(statsDir, "stats.json"), "utf8"), +) as StudySummary[]; +const result = forwardAcceptance(summaries); +jsonFile(join(statsDir, "acceptance.json"), result); +console.log( + `任务书16总体程序验收${result.pass ? "通过" : "未通过"}；详见 ${join(statsDir, "acceptance.json")}；质量判断仍待独立阅读。`, +); +if (!result.pass) process.exitCode = 1; diff --git a/eval/forward/run.ts b/eval/forward/run.ts index 7515e5d..5332dc4 100644 --- a/eval/forward/run.ts +++ b/eval/forward/run.ts @@ -1,5 +1,5 @@ import { execFileSync } from "node:child_process"; -import { appendFileSync, existsSync, mkdirSync } from "node:fs"; +import { appendFileSync, existsSync, mkdirSync, writeFileSync } from "node:fs"; import { join, resolve } from "node:path"; import { forwardAccounts, frozenConfig, jsonFile } from "../lib/forward.ts"; import { ofKind, readLedger } from "../lib/ledger.ts"; @@ -40,6 +40,10 @@ const config = name, null, ); +const daemonTime = () => + execFileSync("docker", ["info", "--format", "{{.SystemTime}}"], { + encoding: "utf8", + }).trim(); const inspect = (container: string) => JSON.parse( execFileSync("docker", ["inspect", container], { @@ -130,6 +134,8 @@ try { if (started !== undefined) { // 容器名是正式入口记录在结构文档里的约定，不 import flow 内部文件。 const container = `inforsi-${name}-${run}`; + // 从杀前划定历史事件区间，覆盖恢复启动到目标结果落账；不把 spawn 当成恢复已执行。 + const lifecycleSince = daemonTime(); const before = inspect(container); jsonFile( join( @@ -208,20 +214,95 @@ try { afterKill, ); observe("after-kill"); - active = start(null); - const afterResume = inspect(container); jsonFile( join( out, - "container-after-resume.json", + "container-before-resume.json", ), - afterResume, + inspect(container), ); + jsonFile( + join( + out, + "container-events-window.json", + ), + { + since: lifecycleSince, + until: null, + run, + resultSeq: null, + }, + ); + active = start(null); watching = watchTimeline( studyDir, active.child, join(out, "timeline.jsonl"), ); + let targetResult; + while (true) { + targetResult = ofKind( + readLedger(studyDir), + "runResult", + ).find( + (r) => + r.content + .run === + run, + ); + if ( + targetResult !== + undefined || + active.child + .exitCode !== + null || + active.child + .signalCode !== + null + ) + break; + await sleep(100); + } + const lifecycleUntil = daemonTime(); + // Docker CLI 按时间回读，不依赖事件订阅的启动时序；原文全存，stats 核对缓存上限和目标 die。 + const events = execFileSync( + "docker", + [ + "events", + "--since", + lifecycleSince, + "--until", + lifecycleUntil, + "--format", + "{{json .}}", + ], + { encoding: "utf8" }, + ); + writeFileSync( + join( + out, + "container-events.jsonl", + ), + events, + ); + jsonFile( + join( + out, + "container-events-window.json", + ), + { + since: lifecycleSince, + until: lifecycleUntil, + run, + resultSeq: + targetResult?.seq ?? + null, + }, + ); + if (targetResult === undefined) + throw new Error( + "恢复进程没有交出目标结果，生命周期证据不完整；保留现场", + ); hit = true; break; } diff --git a/eval/forward/stats.ts b/eval/forward/stats.ts index 94364d5..48bb2de 100644 --- a/eval/forward/stats.ts +++ b/eval/forward/stats.ts @@ -12,6 +12,12 @@ import { jsonFile, scoreBand, } from "../lib/forward.ts"; +import { + sameContainerThroughResult, + type ContainerEvent, + type ContainerSnapshot, + type LifecycleWindow, +} from "../lib/forward-evidence.ts"; import { finalSubmissions, ofKind, readLedger } from "../lib/ledger.ts"; import { lightCheckUsage } from "../lib/light-checks.ts"; import { referenceScores } from "../lib/reference-scores.ts"; @@ -117,16 +123,32 @@ const summaries = dirs.map((dir) => { if (existsSync(join(dir, "kill.json"))) { const evidence = JSON.parse( readFileSync(join(dir, "kill.json"), "utf8"), - ) as { run: string; candidate: string; beforeKillSeq: number }; - type Container = { - Id: string; - State: { StartedAt: string; Running: boolean }; + ) as { + run: string; + candidate: string; + beforeKillSeq: number; + container: string; }; - const snapshots = ["before", "after-kill", "after-resume"].map( + const snapshots = ["before", "after-kill", "before-resume"].map( (part) => JSON.parse( readFileSync(join(dir, `container-${part}.json`), "utf8"), - ) as Container, + ) as ContainerSnapshot, + ); + const lifecycleWindow = JSON.parse( + readFileSync(join(dir, "container-events-window.json"), "utf8"), + ) as LifecycleWindow; + const lifecycleEvents = readFileSync( + join(dir, "container-events.jsonl"), + "utf8", + ) + .split("\n") + .filter(Boolean) + .map((line) => JSON.parse(line) as ContainerEvent); + const lifecycleChecks = sameContainerThroughResult( + snapshots, + lifecycleEvents, + evidence.container, ); const target = accounts.runs.find( (r) => r.plan.content.run === evidence.run, @@ -144,16 +166,18 @@ const summaries = dirs.map((dir) => { evidence, killSeq, snapshots, + snapshotMeaning: "杀前、杀后、恢复启动前；没有把快照当成恢复完成的证据", + lifecycleWindow, + lifecycleChecks, + lifecycleEvents, charged: target.starts.length, resultSeq: target.results[0]?.seq ?? null, checks: { sameContainer: - snapshots[0]!.State.Running && - snapshots.every( - (c) => - c.Id === snapshots[0]!.Id && - c.State.StartedAt === snapshots[0]!.State.StartedAt, - ), + Object.values(lifecycleChecks).every(Boolean) && + lifecycleWindow.run === evidence.run && + lifecycleWindow.resultSeq === target.results[0]?.seq && + Date.parse(lifecycleWindow.since) < Date.parse(lifecycleWindow.until), chargedOnceNoRerun: target.starts.length === 1 && candidatePlans.length === 1, resultAndContinued: @@ -211,6 +235,6 @@ jsonFile(join(output, "manual-review.json"), { "核对各目录launches.jsonl、machine.jsonl和父会话观察；上一份全部结束后才起下一份；不宣称快照排除任意外部进程", }); console.log( - `统计：${output}；程序检查${summaries.every((s) => s.pass) ? "通过" : "未通过"}；质量判断仍待独立阅读。`, + `统计：${output}；所给 ${summaries.length} 份研究的逐研究检查${summaries.every((s) => s.pass) ? "通过" : "未通过"}，不是任务书16总体验收；总体程序验收另运行 node eval/forward/acceptance.ts ${output}；质量判断仍待独立阅读。`, ); if (!summaries.every((s) => s.pass)) process.exitCode = 1; diff --git a/eval/lib/forward-evidence.ts b/eval/lib/forward-evidence.ts new file mode 100644 index 0000000..3f1c3eb --- /dev/null +++ b/eval/lib/forward-evidence.ts @@ -0,0 +1,125 @@ +import { readFileSync, realpathSync } from "node:fs"; +import { join } from "node:path"; + +export interface ContainerSnapshot { + Id: string; + State: { StartedAt: string; Running: boolean }; +} +export interface ContainerEvent { + Type: string; + Action: string; + Actor: { ID: string; Attributes: { name: string } }; +} +export interface LifecycleWindow { + since: string; + until: string; + run: string; + resultSeq: number; +} + +/** 原始历史事件覆盖杀前到目标结果落账；spawn 后的快照本身不证明恢复没有重启。 */ +export function sameContainerThroughResult( + snapshots: ContainerSnapshot[], + events: ContainerEvent[], + container: string, +) { + const original = snapshots[0]!; + const target = events.filter( + (e) => + e.Type === "container" && + (e.Actor.ID === original.Id || e.Actor.Attributes.name === container), + ); + return { + snapshotsAgree: + original.State.Running && + snapshots.every( + (s) => + s.Id === original.Id && + s.State.StartedAt === original.State.StartedAt, + ), + // Docker 只保留最近 256 条事件；碰到上限不能据此声称整个区间没有重启。 + historyWithinLimit: events.length < 256, + completionObserved: target.some( + (e) => e.Actor.ID === original.Id && e.Action === "die", + ), + noRestartOrReplacement: target.every( + (e) => + e.Actor.ID === original.Id && + !["start", "restart", "create"].includes(e.Action), + ), + }; +} + +export interface StudySummary { + dir: string; + study: string; + pass: boolean; + accounts: { checks: { adoptedAndStopped: boolean } }; + kill: { checks: Record } | null; +} +interface LaunchEvent { + event: "start" | "end"; + pid: number; + at: string; +} + +/** 任务书16的总体程序验收；逐研究 stats 通过不等于三份研究的总体验收通过。 */ +export function forwardAcceptance(summaries: StudySummary[]) { + const intervals = summaries.map((s) => { + const launches = readFileSync(join(s.dir, "launches.jsonl"), "utf8") + .trim() + .split("\n") + .filter(Boolean) + .map((line) => JSON.parse(line) as LaunchEvent); + const paired = + launches.length > 0 && + launches.length % 2 === 0 && + launches.every((e, i) => { + if (i % 2 === 1) return e.event === "end"; + const end = launches[i + 1]; + return ( + e.event === "start" && + end?.event === "end" && + end.pid === e.pid && + Date.parse(e.at) <= Date.parse(end.at) && + (i === 0 || Date.parse(launches[i - 1]!.at) <= Date.parse(e.at)) + ); + }); + return { + study: s.study, + dir: realpathSync(s.dir), + paired, + start: launches[0]?.at ?? null, + end: launches.at(-1)?.at ?? null, + }; + }); + const ordered = [...intervals].sort( + (a, b) => Date.parse(a.start!) - Date.parse(b.start!), + ); + const checks = { + threeDifferentStudies: + summaries.length === 3 && + new Set(summaries.map((s) => s.study)).size === 3 && + new Set(intervals.map((s) => s.dir)).size === 3, + allCompletedAndPassed: summaries.every( + (s) => s.pass && s.accounts.checks.adoptedAndStopped, + ), + successfulKill: summaries.some( + (s) => s.kill !== null && Object.values(s.kill.checks).every(Boolean), + ), + launchIntervalsComplete: intervals.every((s) => s.paired), + studiesDoNotOverlap: ordered.every( + (s, i) => + s.paired && + (i === 0 || Date.parse(ordered[i - 1]!.end!) <= Date.parse(s.start!)), + ), + }; + return { + scope: "任务书16总体程序验收（不替代独立研究质量审阅）", + intervals, + checks, + pass: Object.values(checks).every(Boolean), + limit: + "启动区间含同份研究杀后恢复的间隔；只核对所给三份研究，不据此排除任意外部进程抢占CPU。", + }; +} diff --git a/tests/forward-eval.test.ts b/tests/forward-eval.test.ts index c47667f..4ec80a2 100644 --- a/tests/forward-eval.test.ts +++ b/tests/forward-eval.test.ts @@ -1,6 +1,6 @@ import assert from "node:assert/strict"; import { spawnSync } from "node:child_process"; -import { mkdirSync, readFileSync, writeFileSync } from "node:fs"; +import { mkdirSync, readFileSync, rmSync, writeFileSync } from "node:fs"; import { join } from "node:path"; import { DatabaseSync } from "node:sqlite"; import { test } from "node:test"; @@ -10,6 +10,11 @@ import { FORWARD_STOP, scoreBand, } from "../eval/lib/forward.ts"; +import { + forwardAcceptance, + sameContainerThroughResult, + type StudySummary, +} from "../eval/lib/forward-evidence.ts"; import { lightCheckUsage } from "../eval/lib/light-checks.ts"; import { referenceScores } from "../eval/lib/reference-scores.ts"; import { Ledger } from "../src/ledger/index.ts"; @@ -106,10 +111,11 @@ test("forward 导出/统计：历史代码配对、两次核验一次修复、 JSON.stringify({ run: "forward-1", candidate: "r1-c1", + container: "inforsi-test-forward-1", beforeKillSeq: atKill.at(-1)!.seq, }), ); - for (const part of ["before", "after-kill", "after-resume"]) + for (const part of ["before", "after-kill", "before-resume"]) writeFileSync( join(dir, `container-${part}.json`), JSON.stringify({ @@ -122,6 +128,25 @@ test("forward 导出/统计：历史代码配对、两次核验一次修复、 process: exited, judged: judged(2.5, "main"), }); + writeFileSync( + join(dir, "container-events-window.json"), + JSON.stringify({ + since: "2026-10-04T00:00:01Z", + until: "2026-10-04T00:02:00Z", + run: "forward-1", + resultSeq: ledger.records().at(-1)!.seq, + }), + ); + const event = (action: string) => + JSON.stringify({ + Type: "container", + Action: action, + Actor: { + ID: "same-container", + Attributes: { name: "inforsi-test-forward-1" }, + }, + }); + writeFileSync(join(dir, "container-events.jsonl"), event("die") + "\n"); w.decision("adopt", 1, null, { adopted: "r1-c1" }); ledger.append("event", { event: "halted", @@ -180,6 +205,7 @@ test("forward 导出/统计：历史代码配对、两次核验一次修复、 { encoding: "utf8" }, ); assert.equal(statsResult.status, 0, statsResult.stderr); + assert.match(statsResult.stdout, /逐研究检查通过.*不是任务书16总体验收/); const scores = JSON.parse( readFileSync(join(statsOut, "predictions-scores.json"), "utf8"), ); @@ -194,12 +220,10 @@ test("forward 导出/统计：历史代码配对、两次核验一次修复、 chargedOnceNoRerun: true, resultAndContinued: true, }); + // 三份早期快照完全一致，之后同一容器重启仍必须使验收失败。 writeFileSync( - join(dir, "container-after-resume.json"), - JSON.stringify({ - Id: "replacement", - State: { Running: true, StartedAt: "2026-10-04T00:01:00Z" }, - }), + join(dir, "container-events.jsonl"), + [event("die"), event("start"), event("restart"), event("die")].join("\n"), ); const badOut = join(root, "bad-stats"); const bad = spawnSync( @@ -213,6 +237,22 @@ test("forward 导出/统计：历史代码配对、两次核验一次修复、 .sameContainer, false, ); + rmSync(join(dir, "kill.json")); + const singleOut = join(root, "single-without-kill"); + const single = spawnSync( + process.execPath, + ["eval/forward/stats.ts", singleOut, refs, dir], + { encoding: "utf8" }, + ); + assert.equal(single.status, 0, single.stderr); + assert.match( + single.stdout, + /所给 1 份研究的逐研究检查通过.*不是任务书16总体验收/, + ); + assert.equal( + JSON.parse(readFileSync(join(singleOut, "stats.json"), "utf8"))[0].kill, + null, + ); }); test("forward 账目：没启动退还、修复用尽不扣次；缺不跑事件或结果不能通过", (t) => { @@ -352,3 +392,159 @@ test("forward 参考分数：incomplete 原值保留但有效分数为空", (t) assert.equal(rows[0]!.adopted, false); assert.equal(rows[0]!.observationSeq, 2); }); + +test("forward 生命周期：同名替换、缺完成事件、历史缓存触顶都不能证明没有重启", () => { + const snapshots = [ + { Id: "old", State: { Running: true, StartedAt: "before" } }, + ]; + const event = (id: string, action: string) => ({ + Type: "container", + Action: action, + Actor: { ID: id, Attributes: { name: "target" } }, + }); + assert.equal( + sameContainerThroughResult(snapshots, [event("old", "die")], "target") + .noRestartOrReplacement, + true, + ); + assert.equal( + sameContainerThroughResult( + [...snapshots, { Id: "replacement", State: snapshots[0]!.State }], + [event("old", "die")], + "target", + ).snapshotsAgree, + false, + ); + assert.equal( + sameContainerThroughResult( + snapshots, + [event("old", "die"), event("new", "create")], + "target", + ).noRestartOrReplacement, + false, + ); + assert.equal( + sameContainerThroughResult(snapshots, [], "target").completionObserved, + false, + ); + assert.equal( + sameContainerThroughResult( + snapshots, + Array.from({ length: 256 }, () => event("old", "die")), + "target", + ).historyWithinLimit, + false, + ); +}); + +test("forward 总体验收：必须三份不同完成研究、成功kill和不重叠的完整启动区间", (t) => { + const root = tempDir(t); + const launchFile = (dir: string, start: string, end: string) => + writeFileSync( + join(dir, "launches.jsonl"), + [ + { event: "start", pid: 1, at: start }, + { event: "end", pid: 1, at: end }, + ] + .map((e) => JSON.stringify(e)) + .join("\n"), + ); + const summaries: StudySummary[] = [0, 1, 2].map((i) => { + const dir = join(root, `study-${i}`); + mkdirSync(dir); + launchFile(dir, `2026-10-04T0${i}:00:00Z`, `2026-10-04T0${i}:30:00Z`); + return { + dir, + study: `study-${i}`, + pass: true, + accounts: { checks: { adoptedAndStopped: true } }, + kill: + i === 0 + ? { + checks: { + sameContainer: true, + chargedOnceNoRerun: true, + resultAndContinued: true, + }, + } + : null, + }; + }); + // 杀后恢复的进程对也必须完整；间隔仍属于该研究，不能让另一研究插入。 + writeFileSync( + join(summaries[0]!.dir, "launches.jsonl"), + [ + { event: "start", pid: 1, at: "2026-10-04T00:00:00Z" }, + { event: "end", pid: 1, at: "2026-10-04T00:10:00Z" }, + { event: "start", pid: 2, at: "2026-10-04T00:15:00Z" }, + { event: "end", pid: 2, at: "2026-10-04T00:30:00Z" }, + ] + .map((e) => JSON.stringify(e)) + .join("\n"), + ); + assert.equal(forwardAcceptance(summaries).pass, true); + assert.equal(forwardAcceptance(summaries.slice(0, 1)).pass, false); + assert.equal(forwardAcceptance(summaries.slice(0, 2)).pass, false); + assert.equal( + forwardAcceptance(summaries.map((s) => ({ ...s, kill: null }))).pass, + false, + ); + assert.equal( + forwardAcceptance([summaries[0]!, summaries[0]!, summaries[2]!]).pass, + false, + ); + assert.equal( + forwardAcceptance( + summaries.map((s) => ({ + ...s, + accounts: { checks: { adoptedAndStopped: false } }, + })), + ).pass, + false, + ); + assert.equal( + forwardAcceptance( + summaries.map((s) => + s.kill === null + ? s + : { ...s, kill: { checks: { sameContainer: false } } }, + ), + ).pass, + false, + ); + const statsDir = join(root, "stats"); + mkdirSync(statsDir); + writeFileSync(join(statsDir, "stats.json"), JSON.stringify(summaries)); + const passed = spawnSync( + process.execPath, + ["eval/forward/acceptance.ts", statsDir], + { encoding: "utf8" }, + ); + assert.equal(passed.status, 0, passed.stderr); + assert.equal( + JSON.parse(readFileSync(join(statsDir, "acceptance.json"), "utf8")).pass, + true, + ); + writeFileSync( + join(statsDir, "stats.json"), + JSON.stringify(summaries.slice(0, 1)), + ); + const failed = spawnSync( + process.execPath, + ["eval/forward/acceptance.ts", statsDir], + { encoding: "utf8" }, + ); + assert.equal(failed.status, 1, failed.stderr); + assert.match(failed.stdout, /总体程序验收未通过/); + // 包含杀后恢复的整个研究区间不能与下一研究重叠。 + launchFile(summaries[1]!.dir, "2026-10-04T00:20:00Z", "2026-10-04T01:30:00Z"); + assert.equal(forwardAcceptance(summaries).checks.studiesDoNotOverlap, false); + writeFileSync( + join(summaries[1]!.dir, "launches.jsonl"), + JSON.stringify({ event: "start", pid: 1, at: "2026-10-04T01:00:00Z" }), + ); + assert.equal( + forwardAcceptance(summaries).checks.launchIntervalsComplete, + false, + ); +});