[tab] /p/project1/envcomp/mihir/logs/train/crowd-pilot-wave2-broken-resolutions/qwen_8b_no_lora_bs_32_13062921.log (log)
#!/usr/bin/env bash
#
# SBATCH Template Script for SFT Training
# This script is invoked by the launcher with environment variables set
#
# Required environment variables (set by launcher):
#   - MODEL_SIZE: 1.7b, 4b, or 8b
#   - BATCH_SIZE: 8, 16, or 32
#   - USE_LORA: true or false
#

# #SBATCH --nodes=4  # Set dynamically by launcher
#SBATCH --ntasks-per-node=1
#SBATCH --time=24:00:00
#SBATCH --cpus-per-task=5
#SBATCH --gres=gpu:4
#SBATCH --mem=200GB
#SBATCH --account=envcomp
#SBATCH --nodelist=jwb[0700-0799]
#SBATCH --signal=USR1@60
#SBATCH --output=/p/project1/envcomp/mihir/logs/train/crowd-pilot-wave2-broken-resolutions/%x_%j.log
#SBATCH --error=/p/project1/envcomp/mihir/logs/train/crowd-pilot-wave2-broken-resolutions/%x_%j.log

trap 'echo "Job killed by signal"' USR1

# Print this script for debugging
cat $0

echo "============================================"
echo "SBATCH Environment:"
echo "  MODEL_SIZE: ${MODEL_SIZE}"
echo "  BATCH_SIZE: ${BATCH_SIZE}"
echo "  USE_LORA:   ${USE_LORA}"
echo "  CP_SIZE:    ${CONTEXT_PARALLEL_SIZE}"
echo "  SLURM_JOB_ID: ${SLURM_JOB_ID}"
echo "============================================"

# Load required modules
module load CUDA/12
module load cuDNN/9.5.0.50-CUDA-12
module load NCCL/default-CUDA-12
module load Clang/18.1.8

# Activate virtual environment
source .venv/bin/activate

# Export configuration for the run script
export MODEL_SIZE="${MODEL_SIZE}"
export BATCH_SIZE="${BATCH_SIZE}"
export USE_LORA="${USE_LORA}"
export CONTEXT_PARALLEL_SIZE="${CONTEXT_PARALLEL_SIZE}"

# Run the training script
srun sh slurm/jobs/mihir/juelich/crowd-pilot/2-wave-broken-resolutions/run_template.sh
============================================
SBATCH Environment:
  MODEL_SIZE: 8b
  BATCH_SIZE: 32
  USE_LORA:   false
  CP_SIZE:    8
  SLURM_JOB_ID: 13062921
============================================
Lmod has detected the following error: These module(s) or extension(s) exist
but cannot be loaded as requested: "CUDA/12"
   Try: "module spider CUDA/12" to see how to load the module(s).



Lmod has detected the following error: These module(s) or extension(s) exist
but cannot be loaded as requested: "cuDNN/9.5.0.50-CUDA-12"
   Try: "module spider cuDNN/9.5.0.50-CUDA-12" to see how to load the
module(s).



Lmod has detected the following error: These module(s) or extension(s) exist
but cannot be loaded as requested: "NCCL/default-CUDA-12"
   Try: "module spider NCCL/default-CUDA-12" to see how to load the module(s).



Lmod has detected the following error: These module(s) or extension(s) exist
but cannot be loaded as requested: "Clang/18.1.8"
   Try: "module spider Clang/18.1.8" to see how to load the module(s).



srun: error: jwb0746: task 5: Exited with exit code 1
srun: error: jwb0757: task 8: Terminated
srun: error: jwb0764: task 14: Terminated
srun: error: jwb0748: task 7: Terminated
srun: error: jwb0744: task 3: Terminated
srun: error: jwb0762: task 12: Terminated
srun: error: jwb0761: task 11: Terminated
srun: error: jwb0743: task 2: Terminated
srun: error: jwb0759: task 10: Terminated
srun: error: jwb0747: task 6: Terminated
srun: error: jwb0765: task 15: Terminated
srun: error: jwb0763: task 13: Terminated
srun: error: jwb0745: task 4: Terminated
srun: error: jwb0758: task 9: Terminated
srun: error: jwb0741: task 1: Terminated
srun: error: jwb0739: task 0: Terminated
srun: Force Terminated StepId=13062921.0