feat: add Slurm gpu-sbatch training entry for 4090-server
This commit is contained in:
Executable
+46
@@ -0,0 +1,46 @@
|
||||
#!/usr/bin/env bash
|
||||
# ============================================================================
|
||||
# Video-MME 900 自进化训练 —— 4090-server Slurm(gpu-sbatch)入口
|
||||
# ----------------------------------------------------------------------------
|
||||
# 提交(仓库根目录下,1 卡 200 小时):
|
||||
# gpu-sbatch 1 scripts/train_videomme_slurm.sh --time 200:00:00
|
||||
#
|
||||
# 断点续跑(上一任务超时/被杀后,不删 workspace,从 checkpoint 恢复):
|
||||
# RESUME=1 gpu-sbatch 1 scripts/train_videomme_slurm.sh --time 200:00:00
|
||||
#
|
||||
# 观察:
|
||||
# tail -f slurm-<jobid>.out # Slurm 标准输出(与 logs/ 双写)
|
||||
# tail -f logs/train_videomme.log
|
||||
# gpu-status / gpu-cancel
|
||||
#
|
||||
# 前置(那台机器上须就绪):
|
||||
# 1. 分支 = feat/question-gen-v3(main 是旧代码,没有本训练管线)
|
||||
# 2. .env 存在且 REDIS_URL / LLM API 可从该机访问;REDIS_CACHE_TTL 为正整数
|
||||
# 3. store/seeds/adhoc-baseline/、store/videos/、workspaces/video-split/pools.json 已同步
|
||||
# 4. 全新重跑前:rm -rf workspaces/train-videomme logs/train_videomme.log
|
||||
#
|
||||
# 说明:CUDA_VISIBLE_DEVICES 由 Slurm cgroup 注入,本脚本与内层脚本均不覆盖;
|
||||
# 日志三重保险即时输出(PYTHONUNBUFFERED + python -u + loguru stderr)。
|
||||
# ============================================================================
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
cd "$(dirname "$0")/.."
|
||||
|
||||
echo "== Slurm Job: ${SLURM_JOB_ID:-unknown} | CUDA_VISIBLE_DEVICES=${CUDA_VISIBLE_DEVICES:-unset} =="
|
||||
nvidia-smi || true
|
||||
mkdir -p logs
|
||||
|
||||
if [[ "${RESUME:-0}" == "1" ]]; then
|
||||
# 续跑:跳过 Phase 0(seed 已在)与 --fresh,直接从 workspace checkpoint 恢复
|
||||
export HF_HUB_OFFLINE=1 TRANSFORMERS_OFFLINE=1 PYTHONUNBUFFERED=1
|
||||
set -a; source .env; set +a
|
||||
PYTHON="$(conda run -n Video-Tree-TRM which python)"
|
||||
"${PYTHON}" -u main.py --config config/train_videomme.yaml --resume \
|
||||
2>&1 | tee -a logs/train_videomme.log
|
||||
else
|
||||
# 全新训练:复用自包含实验脚本(Phase 0 建/校验 seed + Phase 1 --fresh)
|
||||
bash scripts/train_videomme.sh 2>&1 | tee logs/train_videomme.log
|
||||
fi
|
||||
|
||||
echo "== 训练进程退出,Job ${SLURM_JOB_ID:-unknown} 结束 =="
|
||||
Reference in New Issue
Block a user