#!/usr/bin/env bash
# ============================================================================
# Video-MME 900 自进化训练 —— 4090-server Slurm(gpu-sbatch)入口
# ----------------------------------------------------------------------------
# 提交(仓库根目录下,1 卡 200 小时):
#   gpu-sbatch 1 scripts/train_videomme_slurm.sh --time 200:00:00
#
# 断点续跑(上一任务超时/被杀后,不删 workspace,从 checkpoint 恢复):
#   RESUME=1 gpu-sbatch 1 scripts/train_videomme_slurm.sh --time 200:00:00
#
# 观察:
#   tail -f slurm-<jobid>.out          # Slurm 标准输出(与 logs/ 双写)
#   tail -f logs/train_videomme.log
#   gpu-status / gpu-cancel
#
# 前置(那台机器上须就绪):
#   1. 分支 = feat/question-gen-v3(main 是旧代码,没有本训练管线)
#   2. .env 存在且 REDIS_URL / LLM API 可从该机访问;REDIS_CACHE_TTL 为正整数
#   3. store/seeds/adhoc-baseline/、store/videos/、workspaces/video-split/pools.json 已同步
#   4. 全新重跑前:rm -rf workspaces/train-videomme logs/train_videomme.log
#
# 说明:CUDA_VISIBLE_DEVICES 由 Slurm cgroup 注入,本脚本与内层脚本均不覆盖;
#      日志三重保险即时输出(PYTHONUNBUFFERED + python -u + loguru stderr)。
# ============================================================================

set -euo pipefail

# sbatch 会把脚本拷贝到 Slurm spool 目录执行,$0 不指向仓库路径——须用
# SLURM_SUBMIT_DIR(提交命令时所在目录,即仓库根)定位;本地直跑时回退 dirname。
cd "${SLURM_SUBMIT_DIR:-$(dirname "$0")/..}"
echo "== 工作目录: $(pwd) =="
[[ -f "config/train_videomme.yaml" ]] || {
  echo "错误: 当前目录不是仓库根(找不到 config/train_videomme.yaml)。请在仓库根目录提交 gpu-sbatch。" >&2
  exit 1
}

echo "== Slurm Job: ${SLURM_JOB_ID:-unknown} | CUDA_VISIBLE_DEVICES=${CUDA_VISIBLE_DEVICES:-unset} =="
nvidia-smi || true
mkdir -p logs

if [[ "${RESUME:-0}" == "1" ]]; then
  # 续跑:跳过 Phase 0(seed 已在)与 --fresh,直接从 workspace checkpoint 恢复
  export HF_HUB_OFFLINE=1 TRANSFORMERS_OFFLINE=1 PYTHONUNBUFFERED=1
  set -a; source .env; set +a
  PYTHON="$(conda run -n Video-Tree-TRM which python)"
  "${PYTHON}" -u main.py --config config/train_videomme.yaml --resume \
    2>&1 | tee -a logs/train_videomme.log
else
  # 全新训练:复用自包含实验脚本(Phase 0 建/校验 seed + Phase 1 --fresh)
  bash scripts/train_videomme.sh 2>&1 | tee logs/train_videomme.log
fi

echo "== 训练进程退出,Job ${SLURM_JOB_ID:-unknown} 结束 =="