eval: 准备正向采纳的真实运行恢复与质量评估

新增单份串行启动入口、首个正向运行杀进程恢复证据、逐候选审阅导出与账目统计;参考只读副本分数对照。456项免费测试通过,尚未启动三份真实验收。同步结构说明与当前工作状态。
This commit is contained in:
2026-10-04 04:25:00 -04:00
parent bf5b3f2358
commit b7be524cdc
10 changed files with 1329 additions and 14 deletions
+77
View File
@@ -0,0 +1,77 @@
import { readdirSync, readFileSync } from "node:fs";
import { join } from "node:path";
/** Pi 会话里的 research_check:请求和结果按 toolCallId 配对,不把超限请求算成执行。 */
export function lightCheckUsage(sessionDir: string, maxCount: number) {
type Part = { type: string; id?: string; name?: string; text?: string };
type Message = {
role: string;
toolCallId?: string;
toolName?: string;
isError?: boolean;
content: Part[];
};
const requests: Array<{ id: string; file: string; line: number }> = [];
const results = new Map<
string,
{ file: string; line: number; message: Message }
>();
const files = readdirSync(sessionDir)
.filter((f) => f.endsWith(".jsonl"))
.sort();
if (files.length === 0) throw new Error(`没有会话文件:${sessionDir}`);
for (const file of files) {
readFileSync(join(sessionDir, file), "utf8")
.split("\n")
.forEach((line, index) => {
if (!line.trim()) return;
const entry = JSON.parse(line) as { type: string; message?: Message };
const m = entry.message;
if (entry.type !== "message" || m === undefined) return;
if (m.role === "assistant") {
for (const p of m.content)
if (p.type === "toolCall" && p.name === "research_check")
requests.push({ id: p.id!, file, line: index + 1 });
} else if (m.role === "toolResult" && m.toolName === "research_check")
results.set(m.toolCallId!, { file, line: index + 1, message: m });
});
}
const checks = requests.map((request) => {
const result = results.get(request.id);
if (result === undefined)
return { request, result: null, status: "missingResult", used: null };
const text = result.message.content
.filter((p) => p.type === "text")
.map((p) => p.text)
.join("\n");
const source = { file: result.file, line: result.line, text };
if (result.message.isError)
return { request, result: source, status: "toolError", used: null };
if (text.startsWith("本次调用的轻量检查已用完"))
return { request, result: source, status: "deniedLimit", used: null };
const value = JSON.parse(text) as {
used: number;
maxCount: number;
seconds: number;
result: unknown;
};
return { request, result: source, status: "executed", used: value.used };
});
const used = checks.flatMap((c) => (c.used === null ? [] : [c.used]));
const unresolved = checks.some(
(c) => c.status === "missingResult" || c.status === "toolError",
);
return {
sessionDir,
maxCount,
requested: checks.length,
executed: used.length,
deniedLimit: checks.filter((c) => c.status === "deniedLimit").length,
// 工具报错或缺结果时不能猜实际执行量,也不当作 0 次通过。
pass:
!unresolved &&
used.length <= maxCount &&
used.every((n, i) => n === i + 1),
checks,
};
}