diff --git a/scripts/train_videomme_slurm.sh b/scripts/train_videomme_slurm.sh new file mode 100755 index 0000000..0a773b9 --- /dev/null +++ b/scripts/train_videomme_slurm.sh @@ -0,0 +1,46 @@ +#!/usr/bin/env bash +# ============================================================================ +# Video-MME 900 自进化训练 —— 4090-server Slurm(gpu-sbatch)入口 +# ---------------------------------------------------------------------------- +# 提交(仓库根目录下,1 卡 200 小时): +# gpu-sbatch 1 scripts/train_videomme_slurm.sh --time 200:00:00 +# +# 断点续跑(上一任务超时/被杀后,不删 workspace,从 checkpoint 恢复): +# RESUME=1 gpu-sbatch 1 scripts/train_videomme_slurm.sh --time 200:00:00 +# +# 观察: +# tail -f slurm-.out # Slurm 标准输出(与 logs/ 双写) +# tail -f logs/train_videomme.log +# gpu-status / gpu-cancel +# +# 前置(那台机器上须就绪): +# 1. 分支 = feat/question-gen-v3(main 是旧代码,没有本训练管线) +# 2. .env 存在且 REDIS_URL / LLM API 可从该机访问;REDIS_CACHE_TTL 为正整数 +# 3. store/seeds/adhoc-baseline/、store/videos/、workspaces/video-split/pools.json 已同步 +# 4. 全新重跑前:rm -rf workspaces/train-videomme logs/train_videomme.log +# +# 说明:CUDA_VISIBLE_DEVICES 由 Slurm cgroup 注入,本脚本与内层脚本均不覆盖; +# 日志三重保险即时输出(PYTHONUNBUFFERED + python -u + loguru stderr)。 +# ============================================================================ + +set -euo pipefail + +cd "$(dirname "$0")/.." + +echo "== Slurm Job: ${SLURM_JOB_ID:-unknown} | CUDA_VISIBLE_DEVICES=${CUDA_VISIBLE_DEVICES:-unset} ==" +nvidia-smi || true +mkdir -p logs + +if [[ "${RESUME:-0}" == "1" ]]; then + # 续跑:跳过 Phase 0(seed 已在)与 --fresh,直接从 workspace checkpoint 恢复 + export HF_HUB_OFFLINE=1 TRANSFORMERS_OFFLINE=1 PYTHONUNBUFFERED=1 + set -a; source .env; set +a + PYTHON="$(conda run -n Video-Tree-TRM which python)" + "${PYTHON}" -u main.py --config config/train_videomme.yaml --resume \ + 2>&1 | tee -a logs/train_videomme.log +else + # 全新训练:复用自包含实验脚本(Phase 0 建/校验 seed + Phase 1 --fresh) + bash scripts/train_videomme.sh 2>&1 | tee logs/train_videomme.log +fi + +echo "== 训练进程退出,Job ${SLURM_JOB_ID:-unknown} 结束 =="