54 lines
2.5 KiB
Bash
Executable File
54 lines
2.5 KiB
Bash
Executable File
#!/usr/bin/env bash
|
|
# ============================================================================
|
|
# Video-MME 900 自进化训练 —— 4090-server Slurm(gpu-sbatch)入口
|
|
# ----------------------------------------------------------------------------
|
|
# 提交(仓库根目录下,1 卡 200 小时):
|
|
# gpu-sbatch 1 scripts/train_videomme_slurm.sh --time 200:00:00
|
|
#
|
|
# 断点续跑(上一任务超时/被杀后,不删 workspace,从 checkpoint 恢复):
|
|
# RESUME=1 gpu-sbatch 1 scripts/train_videomme_slurm.sh --time 200:00:00
|
|
#
|
|
# 观察:
|
|
# tail -f slurm-<jobid>.out # Slurm 标准输出(与 logs/ 双写)
|
|
# tail -f logs/train_videomme.log
|
|
# gpu-status / gpu-cancel
|
|
#
|
|
# 前置(那台机器上须就绪):
|
|
# 1. 分支 = feat/question-gen-v3(main 是旧代码,没有本训练管线)
|
|
# 2. .env 存在且 REDIS_URL / LLM API 可从该机访问;REDIS_CACHE_TTL 为正整数
|
|
# 3. store/seeds/adhoc-baseline/、store/videos/、workspaces/video-split/pools.json 已同步
|
|
# 4. 全新重跑前:rm -rf workspaces/train-videomme logs/train_videomme.log
|
|
#
|
|
# 说明:CUDA_VISIBLE_DEVICES 由 Slurm cgroup 注入,本脚本与内层脚本均不覆盖;
|
|
# 日志三重保险即时输出(PYTHONUNBUFFERED + python -u + loguru stderr)。
|
|
# ============================================================================
|
|
|
|
set -euo pipefail
|
|
|
|
# sbatch 会把脚本拷贝到 Slurm spool 目录执行,$0 不指向仓库路径——须用
|
|
# SLURM_SUBMIT_DIR(提交命令时所在目录,即仓库根)定位;本地直跑时回退 dirname。
|
|
cd "${SLURM_SUBMIT_DIR:-$(dirname "$0")/..}"
|
|
echo "== 工作目录: $(pwd) =="
|
|
[[ -f "config/train_videomme.yaml" ]] || {
|
|
echo "错误: 当前目录不是仓库根(找不到 config/train_videomme.yaml)。请在仓库根目录提交 gpu-sbatch。" >&2
|
|
exit 1
|
|
}
|
|
|
|
echo "== Slurm Job: ${SLURM_JOB_ID:-unknown} | CUDA_VISIBLE_DEVICES=${CUDA_VISIBLE_DEVICES:-unset} =="
|
|
nvidia-smi || true
|
|
mkdir -p logs
|
|
|
|
if [[ "${RESUME:-0}" == "1" ]]; then
|
|
# 续跑:跳过 Phase 0(seed 已在)与 --fresh,直接从 workspace checkpoint 恢复
|
|
export HF_HUB_OFFLINE=1 TRANSFORMERS_OFFLINE=1 PYTHONUNBUFFERED=1
|
|
set -a; source .env; set +a
|
|
PYTHON="$(conda run -n Video-Tree-TRM which python)"
|
|
"${PYTHON}" -u main.py --config config/train_videomme.yaml --resume \
|
|
2>&1 | tee -a logs/train_videomme.log
|
|
else
|
|
# 全新训练:复用自包含实验脚本(Phase 0 建/校验 seed + Phase 1 --fresh)
|
|
bash scripts/train_videomme.sh 2>&1 | tee logs/train_videomme.log
|
|
fi
|
|
|
|
echo "== 训练进程退出,Job ${SLURM_JOB_ID:-unknown} 结束 =="
|