diff --git a/.env.example b/.env.example new file mode 100644 index 0000000..ae4cf11 --- /dev/null +++ b/.env.example @@ -0,0 +1,8 @@ +# 复制为 .env 并填入真实值(.env 已被 gitignore,严禁提交密钥) +# teacher API(OpenAI 兼容格式,DeepSeek / MiniMax 二选一填) +TEACHER_API_BASE=https://api.deepseek.com/v1 +TEACHER_API_KEY= +TEACHER_MODEL=deepseek-chat + +# W&B(仅远程训练需要) +WANDB_API_KEY= diff --git a/requirements-remote.txt b/requirements-remote.txt new file mode 100644 index 0000000..64d30c7 --- /dev/null +++ b/requirements-remote.txt @@ -0,0 +1,5 @@ +# 远程独有依赖(gpu-a800-060):训练与推理重件,本地不装。 +-r requirements.txt +vllm>=0.8 # white-box teacher 服务 + 学生 on-policy 生成 +accelerate # 多卡训练启动 +wandb # 训练指标上报 diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..6fdd796 --- /dev/null +++ b/requirements.txt @@ -0,0 +1,9 @@ +# 两端共用核心依赖(本地 + 远程)。版本策略:先宽松安装,两端跑通后按需冻结。 +torch>=2.6 +transformers>=4.51 # Qwen3 系列需要 4.51+ +datasets +numpy +openai>=1.60 # teacher.py:OpenAI 兼容 API 客户端 +python-dotenv +pytest +ruff diff --git a/scripts/setup_remote.sh b/scripts/setup_remote.sh new file mode 100755 index 0000000..701a844 --- /dev/null +++ b/scripts/setup_remote.sh @@ -0,0 +1,54 @@ +#!/usr/bin/env bash +# 远程机 gpu-a800-060 环境搭建(幂等,可重复执行)。 +# 用法:bash scripts/setup_remote.sh +# 硬约束:根分区仅剩 12G —— 环境、缓存、临时目录一律压到 /data/zym 下。 +set -euo pipefail + +DATA_ROOT=/data/zym +ENV_PATH=$DATA_ROOT/envs/ars-opd +REPO_DIR=$DATA_ROOT/ars-opd-rebuild + +# ---- 0. 所有会写盘的路径全部改道 /data(防根分区被写爆)---- +export HF_ENDPOINT=https://hf-mirror.com # huggingface.co 被墙,走镜像 +export HF_HOME=$DATA_ROOT/hf_cache +export CONDA_PKGS_DIRS=$DATA_ROOT/conda_pkgs # conda 包缓存默认在根分区 +export PIP_CACHE_DIR=$DATA_ROOT/pip_cache # pip 缓存默认在根分区 +export TMPDIR=$DATA_ROOT/tmp # 大 wheel 解压临时目录 +mkdir -p "$DATA_ROOT"/{envs,hf_cache,conda_pkgs,pip_cache,tmp} + +# ---- 1. conda 环境(建在 /data,不建在 ~)---- +if [ ! -d "$ENV_PATH" ]; then + conda create -p "$ENV_PATH" python=3.11 -y +fi + +# ---- 2. 代码(远程只读:clone 走 HTTPS 匿名,更新只 git pull)---- +if [ ! -d "$REPO_DIR" ]; then + git clone https://gitea.iomgaa.online/iomgaa/ars-opd-rebuild.git "$REPO_DIR" +else + git -C "$REPO_DIR" pull +fi + +# ---- 3. 依赖 ---- +conda run -p "$ENV_PATH" pip install -r "$REPO_DIR/requirements.txt" -r "$REPO_DIR/requirements-remote.txt" + +# ---- 4. 环境变量持久化(写入 ~/.bashrc,幂等)---- +if ! grep -q "ars-opd-rebuild env" ~/.bashrc; then + cat >> ~/.bashrc <<'EOF' + +# --- ars-opd-rebuild env --- +export HF_ENDPOINT=https://hf-mirror.com +export HF_HOME=/data/zym/hf_cache +export CONDA_PKGS_DIRS=/data/zym/conda_pkgs +export PIP_CACHE_DIR=/data/zym/pip_cache +alias opd='conda activate /data/zym/envs/ars-opd && cd /data/zym/ars-opd-rebuild' +EOF +fi + +# ---- 5. 验证 ---- +echo "=== 验证 torch/CUDA ===" +conda run -p "$ENV_PATH" python -c "import torch; print('torch', torch.__version__, '| cuda可用:', torch.cuda.is_available(), '| 卡数:', torch.cuda.device_count())" +echo "=== 验证单元测试 ===" +conda run -p "$ENV_PATH" python -m pytest "$REPO_DIR/tests" -q +echo "=== 磁盘检查(根分区不应有明显增长)===" +df -h / /data | tail -2 +echo "全部完成。日常使用:输入 opd 进入环境与目录。"