eval: 准备正向采纳的真实运行恢复与质量评估
新增单份串行启动入口、首个正向运行杀进程恢复证据、逐候选审阅导出与账目统计;参考只读副本分数对照。456项免费测试通过,尚未启动三份真实验收。同步结构说明与当前工作状态。
This commit is contained in:
@@ -0,0 +1,77 @@
|
||||
import { readdirSync, readFileSync } from "node:fs";
|
||||
import { join } from "node:path";
|
||||
|
||||
/** Pi 会话里的 research_check:请求和结果按 toolCallId 配对,不把超限请求算成执行。 */
|
||||
export function lightCheckUsage(sessionDir: string, maxCount: number) {
|
||||
type Part = { type: string; id?: string; name?: string; text?: string };
|
||||
type Message = {
|
||||
role: string;
|
||||
toolCallId?: string;
|
||||
toolName?: string;
|
||||
isError?: boolean;
|
||||
content: Part[];
|
||||
};
|
||||
const requests: Array<{ id: string; file: string; line: number }> = [];
|
||||
const results = new Map<
|
||||
string,
|
||||
{ file: string; line: number; message: Message }
|
||||
>();
|
||||
const files = readdirSync(sessionDir)
|
||||
.filter((f) => f.endsWith(".jsonl"))
|
||||
.sort();
|
||||
if (files.length === 0) throw new Error(`没有会话文件:${sessionDir}`);
|
||||
for (const file of files) {
|
||||
readFileSync(join(sessionDir, file), "utf8")
|
||||
.split("\n")
|
||||
.forEach((line, index) => {
|
||||
if (!line.trim()) return;
|
||||
const entry = JSON.parse(line) as { type: string; message?: Message };
|
||||
const m = entry.message;
|
||||
if (entry.type !== "message" || m === undefined) return;
|
||||
if (m.role === "assistant") {
|
||||
for (const p of m.content)
|
||||
if (p.type === "toolCall" && p.name === "research_check")
|
||||
requests.push({ id: p.id!, file, line: index + 1 });
|
||||
} else if (m.role === "toolResult" && m.toolName === "research_check")
|
||||
results.set(m.toolCallId!, { file, line: index + 1, message: m });
|
||||
});
|
||||
}
|
||||
const checks = requests.map((request) => {
|
||||
const result = results.get(request.id);
|
||||
if (result === undefined)
|
||||
return { request, result: null, status: "missingResult", used: null };
|
||||
const text = result.message.content
|
||||
.filter((p) => p.type === "text")
|
||||
.map((p) => p.text)
|
||||
.join("\n");
|
||||
const source = { file: result.file, line: result.line, text };
|
||||
if (result.message.isError)
|
||||
return { request, result: source, status: "toolError", used: null };
|
||||
if (text.startsWith("本次调用的轻量检查已用完"))
|
||||
return { request, result: source, status: "deniedLimit", used: null };
|
||||
const value = JSON.parse(text) as {
|
||||
used: number;
|
||||
maxCount: number;
|
||||
seconds: number;
|
||||
result: unknown;
|
||||
};
|
||||
return { request, result: source, status: "executed", used: value.used };
|
||||
});
|
||||
const used = checks.flatMap((c) => (c.used === null ? [] : [c.used]));
|
||||
const unresolved = checks.some(
|
||||
(c) => c.status === "missingResult" || c.status === "toolError",
|
||||
);
|
||||
return {
|
||||
sessionDir,
|
||||
maxCount,
|
||||
requested: checks.length,
|
||||
executed: used.length,
|
||||
deniedLimit: checks.filter((c) => c.status === "deniedLimit").length,
|
||||
// 工具报错或缺结果时不能猜实际执行量,也不当作 0 次通过。
|
||||
pass:
|
||||
!unresolved &&
|
||||
used.length <= maxCount &&
|
||||
used.every((n, i) => n === i + 1),
|
||||
checks,
|
||||
};
|
||||
}
|
||||
Reference in New Issue
Block a user