层1/T2: max_tokens 放大至 16384(防截断,上限不计费);并发提到 16;进度行加速度与预计剩余时间
Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
This commit is contained in:
+11
-1
@@ -16,6 +16,7 @@ from __future__ import annotations
|
||||
import json
|
||||
import os
|
||||
import re
|
||||
import time
|
||||
from concurrent.futures import ThreadPoolExecutor, as_completed
|
||||
from pathlib import Path
|
||||
|
||||
@@ -140,6 +141,7 @@ def generate_completions(
|
||||
|
||||
failures: list[tuple[str, str]] = []
|
||||
finished = 0
|
||||
start = time.monotonic()
|
||||
# 写盘收口在主线程(as_completed 消费端),工作线程只跑网络请求——
|
||||
# 多线程同写一个文件句柄会交错损坏 JSONL
|
||||
with open(path, "a", encoding="utf-8") as f:
|
||||
@@ -155,7 +157,15 @@ def generate_completions(
|
||||
finally:
|
||||
finished += 1
|
||||
if finished % 20 == 0 or finished == len(todo):
|
||||
print(f"[teacher] {finished}/{len(todo)} 完成", flush=True)
|
||||
elapsed = time.monotonic() - start
|
||||
rate = finished / elapsed * 60 # 条/分
|
||||
eta = (len(todo) - finished) / rate if rate > 0 else 0
|
||||
print(
|
||||
f"[teacher] {finished}/{len(todo)} 完成 | "
|
||||
f"{rate:.1f} 条/分 | 已用 {elapsed / 60:.1f} 分 | "
|
||||
f"预计剩余 {eta:.0f} 分",
|
||||
flush=True,
|
||||
)
|
||||
record = {
|
||||
"key": key,
|
||||
"completion": completion,
|
||||
|
||||
Reference in New Issue
Block a user