层1/T2: max_tokens 放大至 16384(防截断,上限不计费);并发提到 16;进度行加速度与预计剩余时间

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
This commit is contained in:
2026-07-18 08:20:57 -04:00
parent c5a3b7d0bb
commit 4621ebae31
2 changed files with 17 additions and 7 deletions
+11 -1
View File
@@ -16,6 +16,7 @@ from __future__ import annotations
import json
import os
import re
import time
from concurrent.futures import ThreadPoolExecutor, as_completed
from pathlib import Path
@@ -140,6 +141,7 @@ def generate_completions(
failures: list[tuple[str, str]] = []
finished = 0
start = time.monotonic()
# 写盘收口在主线程(as_completed 消费端),工作线程只跑网络请求——
# 多线程同写一个文件句柄会交错损坏 JSONL
with open(path, "a", encoding="utf-8") as f:
@@ -155,7 +157,15 @@ def generate_completions(
finally:
finished += 1
if finished % 20 == 0 or finished == len(todo):
print(f"[teacher] {finished}/{len(todo)} 完成", flush=True)
elapsed = time.monotonic() - start
rate = finished / elapsed * 60 # 条/分
eta = (len(todo) - finished) / rate if rate > 0 else 0
print(
f"[teacher] {finished}/{len(todo)} 完成 | "
f"{rate:.1f} 条/分 | 已用 {elapsed / 60:.1f} 分 | "
f"预计剩余 {eta:.0f}",
flush=True,
)
record = {
"key": key,
"completion": completion,