From f1b6d1f66811a56abd130e499dc1b104da0e3729 Mon Sep 17 00:00:00 2001 From: iomgaa Date: Sun, 19 Jul 2026 05:58:16 -0400 Subject: [PATCH] =?UTF-8?q?=E4=BF=AE=E5=A4=8D:=20=E7=A6=81=E7=94=A8=20HF?= =?UTF-8?q?=20Xet=EF=BC=88hf-mirror=20=E4=B8=8D=E4=BB=A3=E7=90=86=20CAS?= =?UTF-8?q?=EF=BC=8Cteacher=204B=20=E4=B8=8B=E8=BD=BD=20401=EF=BC=89?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 远程冒烟实撞：hf-mirror 供小文件正常，但大权重走 Xet 协议会绕过镜像直连 cas-server.xethub.hf.co 并返 401 Unauthorized。加 HF_HUB_DISABLE_XET=1 退回 经典 HTTP/LFS 下载路径（镜像支持）。两个训练脚本都加，防层 1 换机重下也撞。 Co-Authored-By: Claude Opus 4.8 (1M context) --- scripts/train_sft.sh | 2 ++ scripts/train_whitebox.sh | 4 ++++ 2 files changed, 6 insertions(+) diff --git a/scripts/train_sft.sh b/scripts/train_sft.sh index 6f5e97d..2d331dc 100644 --- a/scripts/train_sft.sh +++ b/scripts/train_sft.sh @@ -22,5 +22,7 @@ export PYTHONUNBUFFERED=1 # 禁止日志缓存（CLAUDE.md §5 export PYTORCH_ALLOC_CONF=expandable_segments:True # 变长序列 batch 易碎片化，按需扩段 export HF_ENDPOINT=${HF_ENDPOINT:-https://hf-mirror.com} export HF_HOME=${HF_HOME:-/data/zym/hf_cache} # 模型缓存落 /data，根分区已满 +# hf-mirror 不代理 HF Xet CAS（大权重走 Xet 会 401，见 train_whitebox.sh 详注） +export HF_HUB_DISABLE_XET=1 torchrun --nproc_per_node=4 --master_port=29571 scripts/train_sft.py "$MODE" diff --git a/scripts/train_whitebox.sh b/scripts/train_whitebox.sh index 6b3f528..ee46507 100755 --- a/scripts/train_whitebox.sh +++ b/scripts/train_whitebox.sh @@ -27,5 +27,9 @@ export PYTHONUNBUFFERED=1 # 禁止日志缓存（CLAUDE.md §5 export PYTORCH_ALLOC_CONF=expandable_segments:True # 变长生成序列易碎片化，按需扩段 export HF_ENDPOINT=${HF_ENDPOINT:-https://hf-mirror.com} export HF_HOME=${HF_HOME:-/data/zym/hf_cache} # 模型缓存落 /data，根分区已满 +# 非显然坑：hf-mirror 不代理 HF 的 Xet CAS——大权重走 Xet 会直连 +# cas-server.xethub.hf.co 并返 401（2026-07-19 teacher 4B 下载实撞）。禁用 Xet +# 退回经典 HTTP/LFS 下载（镜像支持）。若仍不行：pip uninstall hf_xet +export HF_HUB_DISABLE_XET=1 torchrun --nproc_per_node=4 --master_port=29572 scripts/train_whitebox.py "$MODE"