feat: add Slurm gpu-sbatch training entry for 4090-server

This commit is contained in:
2026-07-17 06:02:39 -04:00
parent b0be1f1ae5
commit 9bb09a44d4
+46
View File
@@ -0,0 +1,46 @@
#!/usr/bin/env bash
# ============================================================================
# Video-MME 900 自进化训练 —— 4090-server Slurm(gpu-sbatch)入口
# ----------------------------------------------------------------------------
# 提交(仓库根目录下,1 卡 200 小时):
# gpu-sbatch 1 scripts/train_videomme_slurm.sh --time 200:00:00
#
# 断点续跑(上一任务超时/被杀后,不删 workspace,从 checkpoint 恢复):
# RESUME=1 gpu-sbatch 1 scripts/train_videomme_slurm.sh --time 200:00:00
#
# 观察:
# tail -f slurm-<jobid>.out # Slurm 标准输出(与 logs/ 双写)
# tail -f logs/train_videomme.log
# gpu-status / gpu-cancel
#
# 前置(那台机器上须就绪):
# 1. 分支 = feat/question-gen-v3(main 是旧代码,没有本训练管线)
# 2. .env 存在且 REDIS_URL / LLM API 可从该机访问;REDIS_CACHE_TTL 为正整数
# 3. store/seeds/adhoc-baseline/、store/videos/、workspaces/video-split/pools.json 已同步
# 4. 全新重跑前:rm -rf workspaces/train-videomme logs/train_videomme.log
#
# 说明:CUDA_VISIBLE_DEVICES 由 Slurm cgroup 注入,本脚本与内层脚本均不覆盖;
# 日志三重保险即时输出(PYTHONUNBUFFERED + python -u + loguru stderr)。
# ============================================================================
set -euo pipefail
cd "$(dirname "$0")/.."
echo "== Slurm Job: ${SLURM_JOB_ID:-unknown} | CUDA_VISIBLE_DEVICES=${CUDA_VISIBLE_DEVICES:-unset} =="
nvidia-smi || true
mkdir -p logs
if [[ "${RESUME:-0}" == "1" ]]; then
# 续跑:跳过 Phase 0(seed 已在)与 --fresh,直接从 workspace checkpoint 恢复
export HF_HUB_OFFLINE=1 TRANSFORMERS_OFFLINE=1 PYTHONUNBUFFERED=1
set -a; source .env; set +a
PYTHON="$(conda run -n Video-Tree-TRM which python)"
"${PYTHON}" -u main.py --config config/train_videomme.yaml --resume \
2>&1 | tee -a logs/train_videomme.log
else
# 全新训练:复用自包含实验脚本(Phase 0 建/校验 seed + Phase 1 --fresh)
bash scripts/train_videomme.sh 2>&1 | tee logs/train_videomme.log
fi
echo "== 训练进程退出,Job ${SLURM_JOB_ID:-unknown} 结束 =="