#!/bin/bash
set -ex

## AMD provides RPM packages, so in theory no need to use tarballs,
## but there is no way to get around the licence section at the website
## https://developer.amd.com/amd-aocl/#downloads
## there are two options, gcc 11.1 or AOCC3.2
## aocl-linux-aocc- 3.1.0-1.x86_64.rpm
## aocl-linux-gcc-3.1.0-1.x86_64.rpm
## do be able to download you need to agree to licence
#
# Additionally, AMD provides the Spack (https://spack.io/) recipes for optimally installing BLIS,
# libFLAME, ScaLAPACK, LibM, FFTW, and Sparse libraries

INSTALL_PREFIX=/opt/amd
mkdir -p ${INSTALL_PREFIX}

TARBALL=$(basename $AOCL_DOWNLOAD_URL)

# TODO: this seems a workaround to accept the licence prior download
# should be fixed in readme and made be more general e.g. prior download of the rpm and not the tarball
$COMMON_DIR/download_and_verify.sh $AOCL_DOWNLOAD_URL $AOCL_CHKSUM
tar -xvf ${TARBALL}

cd aocl-linux-aocc-${AOCL_VERSION}
./install.sh -t amd -l blis fftw libflame -i lp64
cp -r amd/${AOCL_VERSION}/* ${INSTALL_PREFIX}
cd .. && rm -rf aocl-linux-aocc-${AOCL_VERSION} ${TARBALL}

$COMMON_DIR/write_component_version.sh "AOCL" ${AOCL_VERSION}

# Setup module files for AMD Libraries
# SUSE HPC uses lmod by default
mkdir -p ${MODULE_FILES_DIRECTORY}/amd/

cat << EOF >> ${MODULE_FILES_DIRECTORY}/amd/aocl-${AOCL_VERSION}

#%Module 1.0
#
#  AOCL
#
prepend-path    LD_LIBRARY_PATH   ${INSTALL_PREFIX}/lib
setenv          AMD_FFTW_INCLUDE  ${INSTALL_PREFIX}/include
EOF

# Create symlinks for modulefiles
ln -sf $(readlink --canonicalize ${MODULE_FILES_DIRECTORY}/amd/aocl-${AOCL_VERSION}) ${MODULE_FILES_DIRECTORY}/amd/aocl


#!/bin/bash
set -ex

# Install DCGM

# workaround to create group for serviceUser=nvidia-dcgm
# as the package expects that the variable "USERGROUPS_ENAB" in /etc/login.defs is set to yes
# in oder that the useradd create the group too, but the SUSE default is "no", so we would change the
# variable or simply use the parameter -U
# wrong cmd: useradd -r -M -s /usr/sbin/nologin ${serviceUser}
# right cmd: useradd -r -M -U -s /usr/sbin/nologin ${serviceUser}
# see: man useradd
# bug reported to Nvidia
serviceUser="nvidia-dcgm"

#check if user exists, if not create it
if ! id $serviceUser &>/dev/null; then
   useradd -r -M -U -s /usr/sbin/nologin ${serviceUser}
fi
zypper --non-interactive install -y -l datacenter-gpu-manager = ${DCGM_VERSION}

systemctl --now enable nvidia-dcgm

# Check if the service is active
systemctl is-active --quiet nvidia-dcgm
error_code=$?
if [ ${error_code} -ne 0 ]
then
    echo "DCGM is inactive!"
    exit ${error_code}
fi

# to verify the installation we can query the system
# You should see a listing of all supported GPUs (and any NVSwitches) found in the system:
# dcgmi discovery -l

$COMMON_DIR/write_component_version.sh "DCGM" ${DCGM_VERSION}


#!/bin/bash
set -ex

# Install docker
zypper --non-interactive in -y -l docker

DOCKER_VERSION=$(rpm -q --qf="%{VERSION}" docker)

# Ensure the Docker service is running
systemctl --now enable docker

# if experimental is needed
#zypper modifyrepo --enable libnvidia-container-experimental

zypper --non-interactive install -y -l --replacefiles nvidia-docker2 nvidia-container-runtime

systemctl restart docker

# Test with
#docker run --rm --gpus all nvidia/cuda:11.0.3-base-ubuntu20.04 nvidia-smi

# Write the docker version to components file
$COMMON_DIR/write_component_version.sh "NVIDIA-DOCKER" ${DOCKER_VERSION}


# With SLE HPC 15 SP4 we have gcc7 and gcc11 provided as the base compiler toolchain
# and requires lua-lmod to supply environment module support.
# the default is gnu/7 within package gnu-compilers-hpc, gcc11 is in gnu11-compilers-hpc
# MODULE_FILES_DIRECTORY=/usr/share/lmod/modulefiles

zypper in -y gnu${GNU_COMPILER_VERSION}-compilers-hpc-devel

$COMMON_DIR/write_component_version.sh "GCC" ${GNU_COMPILER_VERSION}


#!/bin/bash
set -ex

# IntelÂ® oneAPI Math Kernel Library

# Intel provides oneapi RPM packages for SUSE
# and the repository is set up in install_utils.sh
# so we can simply install the needed package

# workaround for wrong multi-version repos of intel.
# instead of use the version for a package search and install
# we need to provide a fixed name including the version
# so instead of
# zypper install -y -l intel-oneapi-mkl = $INTEL_ONE_MKL_VERSION
# we forced to use
zypper --non-interactive install -y -l intel-oneapi-mkl-$INTEL_ONE_MKL_VERSION

$COMMON_DIR/write_component_version.sh "INTEL_ONE_MKL" $INTEL_ONE_MKL_VERSION


#!/bin/bash
set -ex

# SLES comes with mellanox inbox (kernel) drivers by default, so no need to install anything
#

# the ibdev2netdev is only in the external mellanox package, so we do not have it with inbox drivers
wget https://raw.githubusercontent.com/Mellanox/container_scripts/master/ibdev2netdev
mv ibdev2netdev /usr/local/bin
chmod +x /usr/local/bin/ibdev2netdev

# IF you want the external drivers provided by https://network.nvidia.com/products/infiniband-drivers/linux/mlnx_ofed/
# it provides packages for SLES with MLNX_OFED_LINUX-5.7-1.0.2.0-sles15sp4-x86_64.tgz
# SHA256: f3af9dd691dc07404fa07a1c3819de14361dc292d90a9b81aac6a7c729a2ea0f
# you need to agree to the eula and provide the file somewhere

#MLNX_OFED_DOWNLOAD_URL=https://content.mellanox.com/ofed/MLNX_OFED-5.7-1.0.2.0/MLNX_OFED_LINUX-5.7-1.0.2.0-sles15sp4-x86_64.tgz
#TARBALL=$(basename ${MLNX_OFED_DOWNLOAD_URL})
#MOFED_FOLDER=$(basename ${MLNX_OFED_DOWNLOAD_URL} .tgz)

#$COMMON_DIR/download_and_verify.sh $MLNX_OFED_DOWNLOAD_URL "f3af9dd691dc07404fa07a1c3819de14361dc292d90a9b81aac6a7c729a2ea0f"
#tar zxvf ${TARBALL}

# SUSE - if you use the tarball
# The tarball contains modules for SLES default kernel already, there is no need for adding additional parameters to the installscript
# but our default for SLES HPC is the -azure kernel, so we need to add it or switch to kernel-default.

# SUSE default kernel would be simply:
#./${MOFED_FOLDER}/mlnxofedinstall

# SUSE azure kernel (manual check before if kernel and kernel-src fit together, could be not the same due to updates)
#KERNEL=$(uname -r)
#zypper in -y -l kernel-azure-devel kernel-source-azure
#make -C /usr/src/linux-azure oldconfig
#./${MOFED_FOLDER}/mlnxofedinstall --kernel $KERNEL --kernel-sources /usr/src/linux-${KERNEL} --add-kernel-support --skip-repo


#!/bin/bash
set -ex

# SLE HPC 15 SP4 comes with three different implementation of the Message Passing Interface (MPI) standard are provided standard with the HPC module:
#   Open MPI 4 (and version 3)
#   MVAPICH2
#   MPICH 4
#   MPICH-ofi 4.0.1
#   openblas
#   Intel MPI Benchmarks
# These packages have been built with full environment module support (LMOD).
# PLEASE have a look at our documentation
# https://documentation.suse.com/sle-hpc/15-SP4/single-html/hpc-guide/#sec-compute-lib

# https://docs.nvidia.com/networking/category/hpcx
# https://docs.nvidia.com/networking/display/HPCXv28

# Load gcc
set CC=/usr/bin/gcc
set GCC=/usr/bin/gcc

#
INSTALL_PREFIX=/opt

# MVAPICH2
# shipped with SLE HPC
zypper --non-interactive install -y mvapich2-gnu-hpc
MV2_VERSION=$(rpm -q  --qf="%{VERSION}" mvapich2-gnu-hpc)
$COMMON_DIR/write_component_version.sh "MVAPICH2" ${MV2_VERSION}

# OpenMPI 4
# shipped with SLE HPC
zypper --non-interactive install -y ${OMPI}-gnu-hpc  lib${OMPI}-gnu-hpc
OMPI_VERSION=$(rpm -q  --qf="%{VERSION}" ${OMPI}-gnu-hpc)
$COMMON_DIR/write_component_version.sh "OMPI" ${OMPI_VERSION}

# HPC-X
TARBALL=$(basename ${HPCX_DOWNLOAD_URL})
HPCX_FOLDER=$(basename ${HPCX_DOWNLOAD_URL} .tbz)
# the web page said checksum is md5 but in reality is sha256
$COMMON_DIR/download_and_verify.sh ${HPCX_DOWNLOAD_URL} ${HPCX_CHKSUM}
tar -xvf ${TARBALL}
rm -rf ${INSTALL_PREFIX}/${HPCX_FOLDER}
mv ${HPCX_FOLDER} ${INSTALL_PREFIX}
HPCX_PATH=${INSTALL_PREFIX}/${HPCX_FOLDER}
$COMMON_DIR/write_component_version.sh "HPCX" $HPCX_VERSION

# Enable Sharpd
#${HPCX_PATH}/sharp/sbin/sharp_daemons_setup.sh -s -d sharpd
#systemctl enable sharpd
#systemctl start sharpd

# Intel MPI
# as there are more versions in the repos we need to select one
# instead of always get the newest
# workaround for wrong multi-version repos of intel.
# instead of use the version for a package search and install
# we need to provide a fixed name including the version
# so instead of
# zypper install -y -l intel-oneapi-mpi = $INTEL_ONE_MKL_VERSION
# we forced to use
zypper install -y -l intel-oneapi-mpi-${INTEL_ONE_MPI_VERSION}
# Create modulesfiles
/opt/intel/oneapi/modulefiles-setup.sh --force

$COMMON_DIR/write_component_version.sh "IMPI_${IMPI_MAJOR}" ${INTEL_ONE_MPI_VERSION}

#
# # Setup module files for MPIs
#

mkdir -p $MODULE_FILES_DIRECTORY/mpi/

#
# # HPC-X
cat << EOF >> $MODULE_FILES_DIRECTORY/mpi/hpcx-${HPCX_VERSION}
#%Module 1.0
#
#  HPCx ${HPCX_VERSION}
#
#module-whatis "Description: Mellanox HPC-Xâ„¢ Software Toolkit"
set version ${HPCX_VERSION}
conflict mpi
module use ${HPCX_PATH}/modulefiles
module load hpcx
EOF

# libraries are build against gnu-7 and not gnu-11, so we need to have the path hardcoded to gnu-7
# Cannot use links because we need to load compiler as well.
# e.g. /usr/share/lmod/moduledeps/gnu-7/mvapich2/2.3.6
# MVAPICH2 -> already provided by suse package, build with gcc7
cat <<EOF >> ${MODULE_FILES_DIRECTORY}/mpi/mvapich2-${MV2_VERSION}
#%Module 1.0
set version ${MV2_VERSION}
conflict mpi
module use /usr/share/lmod/modulefiles
module load gnu/7 mvapich2/${MV2_VERSION}
EOF

# OpenMPI -> already provided by suse package, build with gcc7
cat <<EOF >> ${MODULE_FILES_DIRECTORY}/mpi/openmpi-${OMPI_VERSION}
#%Module 1.0
set version ${OMPI_VERSION}
conflict mpi
module use /usr/share/lmod/modulefiles
module load gnu/7 openmpi/${OMPI_VERSION}
EOF

# Intel oneAPI
# oneapi provides its own modulefiles
ln -sf  $(readlink --canonicalize $INTELLIBS/mpi/${INTEL_ONE_MPI_VERSION}/modulefiles/mpi) ${MODULE_FILES_DIRECTORY}/mpi/impi_${INTEL_ONE_MPI_VERSION}

# # Create symlinks for modulefiles
ln -sf  $(readlink --canonicalize ${MODULE_FILES_DIRECTORY}/mpi/hpcx-${HPCX_VERSION}) ${MODULE_FILES_DIRECTORY}/mpi/hpcx
ln -sf  $(readlink --canonicalize ${MODULE_FILES_DIRECTORY}/mpi/mvapich2-${MV2_VERSION}) ${MODULE_FILES_DIRECTORY}/mpi/mvapich2
ln -sf  $(readlink --canonicalize ${MODULE_FILES_DIRECTORY}/mpi/openmpi-${OMPI_VERSION}) ${MODULE_FILES_DIRECTORY}/mpi/openmpi
ln -sf  $(readlink --canonicalize ${MODULE_FILES_DIRECTORY}/mpi/impi_${INTEL_ONE_MPI_VERSION}) ${MODULE_FILES_DIRECTORY}/mpi/impi-${IMPI_MAJOR}

# cleanup downloaded tarballs and other installation files/folders
rm -rf *.tar.gz *offline.sh
rm -rf -- */


#!/bin/bash
set -ex

# Install NCCL
# Optimized primitives for inter-GPU communication.

# add rpm build tools
zypper install -y -l rpm-build rpmdevtools git

CUDA_MAJOR=$( echo ${CUDA_VERSION} | cut -d "." -f 1)
CUDA_MINOR=$( echo ${CUDA_VERSION} | cut -d "." -f 2)

TARBALL=$(basename ${NCCL_DOWNLOAD_URL})

pushd /tmp
wget ${NCCL_DOWNLOAD_URL}
tar -xvf ${TARBALL}

pushd nccl-${NCCL_VERSION}

# if you need to limit the number of parallel runs on smaller machines
#mem=$(cat /proc/meminfo | head -1 | sed -e "s/^[^ ]\+[ ]\+\([^ ]\+\)[ ]\+.*/\\1/")
#core=$(cat /proc/cpuinfo | grep processor | wc -l)
#cnt=$(( a=mem/(512*1024), a < core ? a : core ))
#make -j $cnt src.build NVCC_GENCODE="-gencode=arch=compute_80,code=sm_80"

# You should define NVCC_GENCODE in your environment to the minimal set
# of archs to reduce compile time.
#make -j src.build NVCC_GENCODE="-gencode=arch=compute_80,code=sm_80"

make -j src.build

# build rpm packages
make pkg.redhat.build
rpm -i ./build/pkg/rpm/x86_64/libnccl-${NCCL_VERSION}+cuda${CUDA_MAJOR}.${CUDA_MINOR}.x86_64.rpm
rpm -i ./build/pkg/rpm/x86_64/libnccl-devel-${NCCL_VERSION}+cuda${CUDA_MAJOR}.${CUDA_MINOR}.x86_64.rpm
rpm -i ./build/pkg/rpm/x86_64/libnccl-static-${NCCL_VERSION}+cuda${CUDA_MAJOR}.${CUDA_MINOR}.x86_64.rpm
popd
rm -rf nccl-${NCCL_VERSION} $TARBALL

# Install the nccl rdma sharp plugin
# we need the packages: autoconf automake libtool rdma-core-devel
mkdir -p /usr/local/nccl-rdma-sharp-plugins
git clone https://github.com/Mellanox/nccl-rdma-sharp-plugins.git
pushd nccl-rdma-sharp-plugins
./autogen.sh
./configure --prefix=/usr/local/nccl-rdma-sharp-plugins --with-cuda=/usr/local/cuda
make
make install
popd
rm -rf nccl-rdma-sharp-plugins

# Build the nccl tests
source /etc/profile.d/lmod.sh
module load mpi/hpcx
git clone https://github.com/NVIDIA/nccl-tests.git
pushd nccl-tests
make MPI=1 MPI_HOME=${HPCX_MPI_DIR} CUDA_HOME=/usr/local/cuda
popd
mv nccl-tests /opt
module purge
popd

$COMMON_DIR/write_component_version.sh "NCCL" ${NCCL_VERSION}

# Remove installation files
rm -rf /tmp/${TARBALL}
rm -rf /tmp/nccl-${NCCL_VERSION}


#!/bin/bash
set -ex

#
## Nvidia provide certified packages for SLES 15 SP4, so we only need to add the repositories and install the packages
#
DRIVER_BRANCH_VERSION=${NVIDIA_VERSION%.*.*} # branch is like main version e.g. 525 from 525.85.12
CUDA_DASH_VERSION=${CUDA_VERSION/./-}

# to check whats all available in the repo
# zypper se --repo cuda-sles15-x86_64

# install latest cuda package
# the repo contains more versions, so for example we want cuda 11.3 the package name is "cuda-11-3"
#
# cuda 	                    Installs all CUDA Toolkit and Driver packages. Handles upgrading to the next version of the cuda package when it's released.
# cuda-11-8 	            Installs all CUDA Toolkit and Driver packages. Remains at version 11.8 until an additional version of CUDA is installed.
# cuda-toolkit-11-8 	    Installs all CUDA Toolkit packages required to develop CUDA applications. Does not include the driver.
# cuda-tools-11-8 	        Installs all CUDA command line and visual tools.
# cuda-runtime-11-8      	Installs all CUDA Toolkit packages required to run CUDA applications, as well as the Driver packages.
# cuda-compiler-11-8    	Installs all CUDA compiler packages.
# cuda-libraries-11-8 	    Installs all runtime CUDA Library packages.
# cuda-libraries-devel-11-8 	Installs all development CUDA Library packages.
# cuda-drivers 	            Installs all Driver packages. Handles upgrading to the next version of the Driver packages when they're released.

# due to NVIDIA bug in post-install of the nvidia-drivers for kernel-azure, we need to select and install nvidia-gfxG05-kmp-azure manually
# The cuda dependencies select packages with "-default" and then the (wrong) modules for kernel-default instead of kernel-azure got installed
#
# Don't install cuda-drivers: this introduces X11 and Wayland - instead install nvidia-computeGXX
# Don't install cuda-toolkit: this introduces visualization tools
# - instead install cuda-compilers, cuda-command-line-tools, gds-tools and cuda_libraries
zypper -n install -y -l --no-recommends cuda-toolkit-${CUDA_DASH_VERSION} cuda-compiler-${CUDA_DASH_VERSION} cuda-command-line-tools-${CUDA_DASH_VERSION} gds-tools-${CUDA_DASH_VERSION} cuda-libraries-${CUDA_DASH_VERSION}  nvidia-fabricmanager = ${NVIDIA_VERSION} "nvidia-gfxG05-kmp-azure = ${NVIDIA_VERSION}" "nvidia-computeG05 = ${NVIDIA_VERSION}"

$COMMON_DIR/write_component_version.sh "CUDA" ${CUDA_VERSION}
$COMMON_DIR/write_component_version.sh "NVIDIA" ${NVIDIA_VERSION}

# Post-install tasks (version its set through 'alternatives')
echo 'export PATH=$PATH:/usr/local/cuda/bin' | tee -a /etc/bash.bashrc.local
echo '/usr/local/cuda/lib64' | tee /etc/ld.so.conf.d/cuda.conf

# start the fabricmanager - needed for run-tests on ND96asr_v4
# systemctl start nvidia-fabricmanager.service


#!/bin/bash
set -ex

#-------------------------------------------------------------------
# Container Repository
#-------------------------------------------------------------------
# Docker is shipped with SLES by default
# with SLE HPC we need to enable the Container repository
SUSEConnect -p sle-module-containers/${SLE_DOTV}/x86_64

#-------------------------------------------------------------------
# Add SUSE Package Hub
# byacc is only in packagehub
SUSEConnect -p PackageHub/${SLE_DOTV}/x86_64
#-------------------------------------------------------------------

#-------------------------------------------------------------------
# Nvidia provide certified packages for SLES 15, so we only need to add the repositories and install the packages
# add the repo key separately beforehand.
SUSEConnect -p sle-module-NVIDIA-compute/${SLE_MAJOR}/x86_64 --gpg-auto-import-keys
#-------------------------------------------------------------------

# Install pre-reqs and development tools
#

# Add additional repositories

#-------------------------------------------------------------------
# Intel provides oneapi RPM packages for SUSE, so we only need to add the repositories and install the packages
#-------------------------------------------------------------------
# see
# https://www.intel.com/content/www/us/en/develop/documentation/installation-guide-for-intel-oneapi-toolkits-linux/top/installation/install-using-package-managers/yum-dnf-zypper.html

# import package signing keys
rpm --import $INTEL_PUBKEY_URI
# delete if exists
zypper -n rr oneAPI &>/dev/null || :
# add repository
zypper -n addrepo -f -g $INTEL_REPO_URI oneAPI
# fetch key
zypper --non-interactive --gpg-auto-import-keys refresh oneAPI
# disable auto-refresh for the repo (mr -F)
zypper --non-interactive modifyrepo --no-refresh oneAPI

# list all packages
# sudo -E zypper pa -ir oneAPI
#-------------------------------------------------------------------

#-------------------------------------------------------------------
# Nvidia container repo
#-------------------------------------------------------------------
# see https://docs.nvidia.com/datacenter/cloud-native/container-toolkit/install-guide.html
# Check https://nvidia.github.io/libnvidia-container
zypper --non-interactive rr libnvidia-container &>/dev/null || :
zypper addrepo -f -g $NVIDIA_CONTAINER_REPO_URI
# fetch key
zypper --non-interactive --gpg-auto-import-keys refresh libnvidia-container

#-------------------------------------------------------------------
# Add SUSE Package Hub
# byacc is only in packagehub
SUSEConnect -p PackageHub/${SLE_DOTV}/x86_64
#-------------------------------------------------------------------

#
## SLES HPC ship with many HPC packages already, so no need to build it - simple install is enough
#
# Install base compiler (this will pull in packages for HPC and Lmod as well)
zypper --non-interactive in -y gnu-compilers-hpc-devel

#
# If you run kernel-default remove "-azure" from the kernel package names below
#
zypper install -y \
    numactl \
    byacc \
    atk \
    m4 \
    ${KERNEL_VERSION:+kernel-azure-devel = ${KERNEL_VERSION}} \
    ${KERNEL_VERSION:+kernel-source-azure = ${KERNEL_VERSION}} \
    binutils \
    fuse \
    cmake \
    libarchive13 \
    libsecret-1-0 \
    libnuma-devel \
    libibverbs-utils \
    perftest \
    mstflint \
    bzip2 \
    vim-data \
    clone-master-clean-up \
    insserv-compat \
    rpm-build \
    python3-devel\
    patch \
    python-rpm-macros \
    lshw \
    autoconf \
    automake \
    libtool \
    nfs-client \
    jq \
    rdma-core-devel \
    wget

# Install azcopy tool
## To copy blobs or files to or from a storage account.
wget ${AZCOPY_DOWNLOAD_URL}
tar -xvf ${AZTARBALL}
## copy the azcopy to the bin path - better would be ${LOCALBIN}
mv azcopy_linux_amd64_${AZVERSION}/azcopy ${LOCALBIN}
chmod +x ${LOCALBIN}/azcopy
$COMMON_DIR/write_component_version.sh "azcopy" ${AZCOPY_VERSION}
## remove azcopy tarball and directory
rm -rf *.tar.gz azcopy_linux_amd64_${AZVERSION}


#!/bin/bash

# Get the new SUSEConnect
zypper --non-interactive install -y suseconnect-ng


#!/bin/bash
set -ex

# set configuration
source ./config

# set properties
source ./set_properties.sh

# SLE version specific fixes
./fixes.sh

# install utils
../common/install_utils.sh

# install compilers
../common/install_gcc.sh

# install mellanox ofed
../common/install_mellanoxofed.sh

# install mpi libraries
../common/install_mpis.sh

# install nvidia gpu driver
../common/install_nvidiagpudriver.sh

# Install NCCL
../common/install_nccl.sh

# install AMD tuned libraries
../common/install_amd_libs.sh

# install Intel libraries
../common/install_intel_libs.sh

# Install NVIDIA docker container
../common/install_docker.sh

# Install Nvidia Datacenter GPU Manager (DCGM)
../common/install_dcgm.sh

# optimizations
../common/hpc-tuning.sh

# Network Optimization
$COMMON_DIR/network-tuning.sh

# copy test file
$COMMON_DIR/copy_test_file.sh

# install diagnostic script
$COMMON_DIR/install_hpcdiag.sh

# install persistent rdma naming
#
# SUSE ships the rdma core userspace and libraries package by default
# and provides persistent naming through udev rules
# see for example /usr/lib/udev/rules.d/60-rdma-persistent-naming.rules

# cleanup downloaded tarballs
rm -rf *.tgz *.bz2 *.tbz *.tar.gz *.run *.deb *_offline.sh
# cleanup directories
rm -rf -- */

# if you want to use it as golden-image pls. run
#
#/usr/sbin/clone-master-clean-up
#/usr/sbin/waagent -force -deprovision+user && export HISTSIZE=0 && sync


#!/bin/bash

export TOP_DIR=../../..
export COMMON_DIR=../../../common
export TEST_DIR=../../../tests


#!/bin/bash
# transform long form MOFED-LTS flag to short
for arg in "$@"; do
    shift
    case "$arg" in
        "--mofed-lts") set -- "$@" "-l" ;;
        *) set -- "$@" "$arg"
    esac
done

# display the usage of lts flag for user
usage() { echo "Usage: $0 [--mofed-lts <true|false>]"  1>&2; exit 1; }

while getopts ":l:" o; do
    case "${o}" in
        l)
            l=${OPTARG}
            ((l == true || l == false)) || usage
            ;;
        *)
            usage
            ;;
    esac
done
shift $((OPTIND-1))

if [ -z "${l}" ]; then
    usage
fi

MOFED_LTS=${l} # true/ false

source /etc/profile

GCC_VERSION="9.2.0"

# Find distro
find_distro() {
    local os=`cat /etc/os-release | awk 'match($0, /^NAME="(.*)"/, result) { print result[1] }'`
    if [[ $os == "CentOS Linux" ]]
    then
        local centos_distro=`find_centos_distro`
        echo "${os} ${centos_distro}"
    elif [[ $os == "AlmaLinux" ]]
    then
        local alma_distro=`find_alma_distro`
        echo "${os} ${alma_distro}"
    elif [[ $os == "Ubuntu" ]]
    then
        local ubuntu_distro=`find_ubuntu_distro`
        echo "${os} ${ubuntu_distro}"
    elif [[ $os == "SLE_HPC" ]]
    then
        local sle_hpc_distro=`find_sle_hpc_distro`
        echo ${sle_hpc_distro}
    else
        echo "*** Error - invalid distro!"
        exit -1
    fi
}

# Find CentOS distro
find_centos_distro() {
    echo `cat /etc/redhat-release | awk '{print $4}'`
}

# Find Alma distro
find_alma_distro() {
    echo `cat /etc/redhat-release | awk '{print $3}'`
}

# Find Ubuntu distro
find_ubuntu_distro() {
    echo `cat /etc/os-release | awk 'match($0, /^PRETTY_NAME="(.*)"/, result) { print result[1] }' | awk '{print $2}' | cut -d. -f1,2`
}

# Find SUSE Linux Enterprise HPC distro
find_sle_hpc_distro() {
    echo $(cat /etc/os-release | awk 'match($0, /^PRETTY_NAME="(.*)"/, result) { print result[1] }')
}

distro=`find_distro`
echo "Detected distro: ${distro}"

if [ "${MOFED_LTS}" = true ]
then
    HPCX_VERSION_UBUNTU="v2.7.0"
    MOFED_VERSION_UBUNTU="MLNX_OFED_LINUX-4.9-6.0.6.0"
    HPCX_MOFED_INTEGRATION_VERSION="MLNX_OFED_LINUX-4.7-1.0.0.1"
    HPCX_OMB_PATH_UBUNTU_1804="/opt/hpcx-${HPCX_VERSION_UBUNTU}-gcc-${HPCX_MOFED_INTEGRATION_VERSION}-ubuntu18.04-x86_64/ompi/tests/osu-micro-benchmarks-5.6.2"
    IMPI_2021_VERSION_UBUNTU="2021.9.0"
    OMPI_VERSION_UBUNTU="4.1.5"
else
    OMPI_VERSION_UBUNTU="4.1.5"
    HPCX_MOFED_INTEGRATION_VERSION="MLNX_OFED_LINUX-5.4-1.0.3.0"
    case ${distro} in
        "Ubuntu 18.04") HPCX_VERSION_UBUNTU="v2.15";
            MOFED_VERSION_UBUNTU="MLNX_OFED_LINUX-23.04-1.1.3.0";
            IMPI_2021_VERSION_UBUNTU="2021.9.0";
            ;;
        "Ubuntu 20.04") HPCX_VERSION_UBUNTU="v2.15";
            MOFED_VERSION_UBUNTU="MLNX_OFED_LINUX-23.04-1.1.3.0";
            IMPI_2021_VERSION_UBUNTU="2021.9.0";
            ;;
        "Ubuntu 22.04") HPCX_VERSION_UBUNTU="v2.15";
            MOFED_VERSION_UBUNTU="MLNX_OFED_LINUX-23.04-1.1.3.0";
            IMPI_2021_VERSION_UBUNTU="2021.9.0";
            ;;
        *) ;;
    esac
    HPCX_OMB_PATH_UBUNTU_1804="/opt/hpcx-${HPCX_VERSION_UBUNTU}-gcc-MLNX_OFED_LINUX-5-ubuntu18.04-cuda12-gdrcopy2-nccl2.17-x86_64/ompi/tests/osu-micro-benchmarks-5.8"
fi

HPCX_VERSION_CENTOS="v2.9.0"
HPCX_VERSION_SUSE="2.12"

MVAPICH2_VERSION_CENTOS="2.3.6"
MVAPICH2_VERSION_ALMA="2.3.7"
MVAPICH2_VERSION_UBUNTU="2.3.7-1"
MVAPICH2_VERSION_SUSE="2.3.6"

OMPI_VERSION_CENTOS="4.1.1"
OMPI_VERSION_ALMA_86="4.1.3"
OMPI_VERSION_ALMA_87="4.1.5"
OMPI_VERSION_SUSE="4.1.1"

IMPI_2021_VERSION_CENTOS="2021.4.0"
IMPI_2021_VERSION_ALMA_86="2021.7.0"
IMPI_2021_VERSION_ALMA_87="2021.9.0"
IMPI_2021_VERSION_SUSE="2021.9.0"

MVAPICH2X_INSTALLATION_DIRECTORY="/opt/mvapich2-x"
IMPI2018_PATH="/opt/intel/compilers_and_libraries_2018.5.274"

MOFED_VERSION_CENTOS="MLNX_OFED_LINUX-5.4-1.0.3.0"
MOFED_VERSION_CENTOS_79="MLNX_OFED_LINUX-5.4-3.0.0.0"
MOFED_VERSION_CENTOS_83="MLNX_OFED_LINUX-5.2-1.0.4.0"
MOFED_VERSION_ALMA_86="MLNX_OFED_LINUX-5.8-1.0.1.1"
MOFED_VERSION_ALMA_87="MLNX_OFED_LINUX-23.04-0.5.3.3"
MOFED_VERSION_SUSE="MLNX_OFED_INBOX_5.14.21-4.0.0"
#MOFED_VERSION_SUSE="MLNX_OFED-5.7-1.0.2.0"

HPCX_OMB_PATH_CENTOS_76="/opt/hpcx-${HPCX_VERSION_CENTOS}-gcc${GCC_VERSION}-${MOFED_VERSION_CENTOS}-redhat7.6-x86_64/ompi/tests/osu-micro-benchmarks-5.6.2"
HPCX_OMB_PATH_CENTOS_77="/opt/hpcx-${HPCX_VERSION_CENTOS}-gcc${GCC_VERSION}-${MOFED_VERSION_CENTOS}-redhat7.7-x86_64/ompi/tests/osu-micro-benchmarks-5.6.2"
HPCX_OMB_PATH_CENTOS_78="/opt/hpcx-${HPCX_VERSION_CENTOS}-gcc${GCC_VERSION}-${MOFED_VERSION_CENTOS}-redhat7.8-x86_64/ompi/tests/osu-micro-benchmarks-5.6.2"
HPCX_OMB_PATH_CENTOS_79="/opt/hpcx-${HPCX_VERSION_CENTOS}-gcc${GCC_VERSION}-${HPCX_MOFED_INTEGRATION_VERSION}-redhat7.9-x86_64/ompi/tests/osu-micro-benchmarks-5.6.2"
HPCX_OMB_PATH_CENTOS_81="/opt/hpcx-${HPCX_VERSION_CENTOS}-gcc${GCC_VERSION}-${MOFED_VERSION_CENTOS}-redhat8.1-x86_64/ompi/tests/osu-micro-benchmarks-5.6.2"
HPCX_OMB_PATH_CENTOS_83="/opt/hpcx-v2.8.0-gcc-${MOFED_VERSION_CENTOS_83}-redhat8.3-x86_64/ompi/tests/osu-micro-benchmarks-5.6.2"
MODULE_FILES_ROOT_CENTOS="/usr/share/Modules/modulefiles"
IMPI2021_PATH_CENTOS="/opt/intel/oneapi/mpi/${IMPI_2021_VERSION_CENTOS}"
# added "libexec" to the path, as centos and ubuntu use "libexec", but SUSE only "lib"
MVAPICH2_PATH_CENTOS="/opt/mvapich2-${MVAPICH2_VERSION_CENTOS}/libexec"
MVAPICH2X_PATH_CENTOS="${MVAPICH2X_INSTALLATION_DIRECTORY}/gnu9.2.0/mofed5.1/azure-xpmem/mpirun"
OPENMPI_PATH_CENTOS="/opt/openmpi-${OMPI_VERSION_CENTOS}"

HPCX_OMB_PATH_ALMA_86="/opt/hpcx-v2.14-gcc-MLNX_OFED_LINUX-5-redhat8-cuda11-gdrcopy2-nccl2.16-x86_64/ompi/tests/osu-micro-benchmarks-5.8"
HPCX_OMB_PATH_ALMA_87="/opt/hpcx-v2.15-gcc-MLNX_OFED_LINUX-5-redhat8-cuda12-gdrcopy2-nccl2.17-x86_64/ompi/tests/osu-micro-benchmarks-5.8"
MODULE_FILES_ROOT_ALMA="/usr/share/Modules/modulefiles"
IMPI2021_PATH_ALMA_86="/opt/intel/oneapi/mpi/${IMPI_2021_VERSION_ALMA_86}"
IMPI2021_PATH_ALMA_87="/opt/intel/oneapi/mpi/${IMPI_2021_VERSION_ALMA_87}"
# added "libexec" to the path, as rh+clones and ubuntu use "libexec", but SUSE only "lib"
MVAPICH2_PATH_ALMA="/opt/mvapich2-${MVAPICH2_VERSION_ALMA}/libexec"
OPENMPI_PATH_ALMA_86="/opt/openmpi-${OMPI_VERSION_ALMA_86}"
OPENMPI_PATH_ALMA_87="/opt/openmpi-${OMPI_VERSION_ALMA_87}"

MODULE_FILES_ROOT_UBUNTU="/usr/share/modules/modulefiles"
HPCX_OMB_PATH_UBUNTU_2004="/opt/hpcx-${HPCX_VERSION_UBUNTU}-gcc-MLNX_OFED_LINUX-5-ubuntu20.04-cuda12-gdrcopy2-nccl2.17-x86_64/ompi/tests/osu-micro-benchmarks-5.8"
HPCX_OMB_PATH_UBUNTU_2204="/opt/hpcx-${HPCX_VERSION_UBUNTU}-gcc-MLNX_OFED_LINUX-5-ubuntu22.04-cuda12-gdrcopy2-nccl2.17-x86_64/ompi/tests/osu-micro-benchmarks-5.8"
IMPI2021_PATH_UBUNTU="/opt/intel/oneapi/mpi/${IMPI_2021_VERSION_UBUNTU}"
# added "libexec" to the path, as centos and ubuntu use "libexec", but SUSE only "lib"
MVAPICH2_PATH_UBUNTU="/opt/mvapich2-${MVAPICH2_VERSION_UBUNTU}/libexec"
MVAPICH2X_PATH_UBUNTU="${MVAPICH2X_INSTALLATION_DIRECTORY}/gnu9.2.0/mofed5.0/advanced-xpmem/mpirun"
OPENMPI_PATH_UBUNTU="/opt/openmpi-${OMPI_VERSION_UBUNTU}"

MODULE_FILES_ROOT_SUSE="/usr/share/lmod/modulefiles"
HPCX_OMB_PATH_SUSE="/opt/hpcx-v${HPCX_VERSION_SUSE}-gcc-inbox-suse15.4-cuda11-gdrcopy2-nccl${HPCX_VERSION_SUSE}-x86_64/ompi/tests/osu-micro-benchmarks-5.8"
IMPI2021_PATH_SUSE="/opt/intel/oneapi/mpi/${IMPI_2021_VERSION_SUSE}"
# SUSE use lib instead of libexec
MVAPICH2_PATH_SUSE="/usr/lib/hpc/gnu7/mpi/mvapich2-psm2/2.3.6/lib"
MVAPICH2X_PATH_SUSE=""
OPENMPI_PATH_SUSE="/usr/lib/hpc/gnu7/mpi/openmpi/4.1.1"

CHECK_HPCX=0
CHECK_IMPI_2021=0
CHECK_IMPI_2018=0
CHECK_OMPI=0
CHECK_MVAPICH2=0
CHECK_MVAPICH2X=0
CHECK_CUDA=0
CHECK_AOCL=1
CHECK_NCCL=0
CHECK_GCC=1
CHECK_DOCKER=0

if [[ $distro == *"CentOS Linux"* ]]
then
    MKL_VERSION="2021.1.1"
elif [[ $distro == "Ubuntu"* ]]
then
    MKL_VERSION="2023.1.0"
elif [[ $distro == "AlmaLinux 8.6" ]]
then
    MKL_VERSION="2022.1.0"
else
    MKL_VERSION="2023.1.0"
fi

if [[ $distro == "CentOS Linux 7.6.1810" ]]
then
    HPCX_OMB_PATH=${HPCX_OMB_PATH_CENTOS_76}
    CHECK_HPCX=1
    CHECK_IMPI_2021=1
    CHECK_IMPI_2018=1
    CHECK_OMPI=1
    CHECK_MVAPICH2=1
    CHECK_MVAPICH2X=0
    MODULE_FILES_ROOT=${MODULE_FILES_ROOT_CENTOS}
    MOFED_VERSION=${MOFED_VERSION_CENTOS}
    IMPI2021_PATH=${IMPI2021_PATH_CENTOS}
    MVAPICH2_PATH=${MVAPICH2_PATH_CENTOS}
    MVAPICH2X_PATH=${MVAPICH2X_PATH_CENTOS}
    OPENMPI_PATH=${OPENMPI_PATH_CENTOS}
elif [[ $distro == "CentOS Linux 7.7.1908" ]]
then
    HPCX_OMB_PATH=${HPCX_OMB_PATH_CENTOS_77}
    CHECK_HPCX=1
    CHECK_IMPI_2021=1
    CHECK_IMPI_2018=1
    CHECK_OMPI=1
    CHECK_MVAPICH2=1
    CHECK_MVAPICH2X=0
    MODULE_FILES_ROOT=${MODULE_FILES_ROOT_CENTOS}
    MOFED_VERSION=${MOFED_VERSION_CENTOS}
    IMPI2021_PATH=${IMPI2021_PATH_CENTOS}
    MVAPICH2_PATH=${MVAPICH2_PATH_CENTOS}
    MVAPICH2X_PATH=${MVAPICH2X_PATH_CENTOS}
    OPENMPI_PATH=${OPENMPI_PATH_CENTOS}
elif [[ $distro == "CentOS Linux 7.8.2003" ]]
then
    HPCX_OMB_PATH=${HPCX_OMB_PATH_CENTOS_78}
    CHECK_HPCX=1
    CHECK_IMPI_2021=1
    CHECK_IMPI_2018=1
    CHECK_OMPI=1
    CHECK_MVAPICH2=1
    CHECK_MVAPICH2X=0
    MODULE_FILES_ROOT=${MODULE_FILES_ROOT_CENTOS}
    MOFED_VERSION=${MOFED_VERSION_CENTOS}
    IMPI2021_PATH=${IMPI2021_PATH_CENTOS}
    MVAPICH2_PATH=${MVAPICH2_PATH_CENTOS}
    MVAPICH2X_PATH=${MVAPICH2X_PATH_CENTOS}
    OPENMPI_PATH=${OPENMPI_PATH_CENTOS}
elif [[ $distro == "CentOS Linux 7.9.2009" ]]
then
    HPCX_OMB_PATH=${HPCX_OMB_PATH_CENTOS_79}
    CHECK_HPCX=1
    CHECK_IMPI_2021=1
    CHECK_IMPI_2018=1
    CHECK_OMPI=1
    CHECK_MVAPICH2=1
    CHECK_MVAPICH2X=0
    CHECK_DOCKER=1
    MODULE_FILES_ROOT=${MODULE_FILES_ROOT_CENTOS}
    MOFED_VERSION=${MOFED_VERSION_CENTOS_79}
    IMPI2021_PATH=${IMPI2021_PATH_CENTOS}
    MVAPICH2_PATH=${MVAPICH2_PATH_CENTOS}
    MVAPICH2X_PATH=${MVAPICH2X_PATH_CENTOS}
    OPENMPI_PATH=${OPENMPI_PATH_CENTOS}
    CHECK_AOCL=1
    CHECK_NCCL=1
elif [[ $distro == "CentOS Linux 8.1.1911" ]]
then
    HPCX_OMB_PATH=${HPCX_OMB_PATH_CENTOS_81}
    CHECK_HPCX=1
    CHECK_IMPI_2021=1
    CHECK_OMPI=1
    CHECK_MVAPICH2=1
    CHECK_MVAPICH2X=0
    MODULE_FILES_ROOT=${MODULE_FILES_ROOT_CENTOS}
    MOFED_VERSION=${MOFED_VERSION_CENTOS}
    IMPI2021_PATH=${IMPI2021_PATH_CENTOS}
    MVAPICH2_PATH=${MVAPICH2_PATH_CENTOS}
    MVAPICH2X_PATH=${MVAPICH2X_PATH_CENTOS}
    OPENMPI_PATH=${OPENMPI_PATH_CENTOS}
elif [[ $distro == "CentOS Linux 8.3.2011" ]]
then
    HPCX_OMB_PATH=${HPCX_OMB_PATH_CENTOS_83}
    CHECK_HPCX=1
    CHECK_IMPI_2021=1
    CHECK_OMPI=1
    CHECK_MVAPICH2=1
    CHECK_MVAPICH2X=0
    MODULE_FILES_ROOT=${MODULE_FILES_ROOT_CENTOS}
    MOFED_VERSION=${MOFED_VERSION_CENTOS}
    IMPI2021_PATH=${IMPI2021_PATH_CENTOS}
    MVAPICH2_PATH=${MVAPICH2_PATH_CENTOS}
    MVAPICH2X_PATH=${MVAPICH2X_PATH_CENTOS}
    OPENMPI_PATH=${OPENMPI_PATH_CENTOS}
elif [[ $distro == "AlmaLinux 8.6" ]]
then
    HPCX_OMB_PATH=${HPCX_OMB_PATH_ALMA_86}
    CHECK_HPCX=1
    CHECK_IMPI_2021=1
    CHECK_IMPI_2018=0
    CHECK_OMPI=1
    CHECK_MVAPICH2=1
    CHECK_MVAPICH2X=0
    MODULE_FILES_ROOT=${MODULE_FILES_ROOT_ALMA}
    MOFED_VERSION=${MOFED_VERSION_ALMA_86}
    IMPI2021_PATH=${IMPI2021_PATH_ALMA_86}
    MVAPICH2_PATH=${MVAPICH2_PATH_ALMA}
    OPENMPI_PATH=${OPENMPI_PATH_ALMA_86}
    CHECK_AOCL=1
    CHECK_NCCL=1
    CHECK_DOCKER=1
elif [[ $distro == "AlmaLinux 8.7" ]]
then
    HPCX_OMB_PATH=${HPCX_OMB_PATH_ALMA_87}
    CHECK_HPCX=1
    CHECK_IMPI_2021=1
    CHECK_IMPI_2018=0
    CHECK_OMPI=1
    CHECK_MVAPICH2=1
    CHECK_MVAPICH2X=0
    MODULE_FILES_ROOT=${MODULE_FILES_ROOT_ALMA}
    MOFED_VERSION=${MOFED_VERSION_ALMA_87}
    IMPI2021_PATH=${IMPI2021_PATH_ALMA_87}
    MVAPICH2_PATH=${MVAPICH2_PATH_ALMA}
    OPENMPI_PATH=${OPENMPI_PATH_ALMA_87}
    CHECK_AOCL=1
    CHECK_NCCL=1
    CHECK_DOCKER=1
elif [[ $distro == "Ubuntu 18.04" ]]
then
    HPCX_OMB_PATH=${HPCX_OMB_PATH_UBUNTU_1804}
    CHECK_HPCX=1
    CHECK_IMPI_2021=1
    CHECK_MVAPICH2=1
    CHECK_OMPI=1
    CHECK_BLIS_MT=1
    MODULE_FILES_ROOT=${MODULE_FILES_ROOT_UBUNTU}
    MOFED_VERSION=${MOFED_VERSION_UBUNTU}
    IMPI2021_PATH=${IMPI2021_PATH_UBUNTU}
    MVAPICH2_PATH=${MVAPICH2_PATH_UBUNTU}
    MVAPICH2X_PATH=${MVAPICH2X_PATH_UBUNTU}
    OPENMPI_PATH=${OPENMPI_PATH_UBUNTU}
    CHECK_AOCL=0
    CHECK_GCC=0
    CHECK_NCCL=1
    CHECK_DOCKER=1
elif [[ $distro == "Ubuntu 20.04" ]]
then
    HPCX_OMB_PATH=${HPCX_OMB_PATH_UBUNTU_2004}
    CHECK_HPCX=1
    CHECK_IMPI_2021=1
    CHECK_MVAPICH2=1
    CHECK_OMPI=1
    CHECK_BLIS_MT=1
    MODULE_FILES_ROOT=${MODULE_FILES_ROOT_UBUNTU}
    MOFED_VERSION=${MOFED_VERSION_UBUNTU}
    IMPI2021_PATH=${IMPI2021_PATH_UBUNTU}
    MVAPICH2_PATH=${MVAPICH2_PATH_UBUNTU}
    MVAPICH2X_PATH=${MVAPICH2X_PATH_UBUNTU}
    OPENMPI_PATH=${OPENMPI_PATH_UBUNTU}
    CHECK_AOCL=0
    CHECK_NCCL=1
    CHECK_GCC=0
    CHECK_DOCKER=1
elif [[ $distro == "Ubuntu 22.04" ]]
then
    HPCX_OMB_PATH=${HPCX_OMB_PATH_UBUNTU_2204}
    CHECK_HPCX=1
    CHECK_IMPI_2021=1
    CHECK_MVAPICH2=1
    CHECK_OMPI=1
    CHECK_BLIS_MT=1
    MODULE_FILES_ROOT=${MODULE_FILES_ROOT_UBUNTU}
    MOFED_VERSION=${MOFED_VERSION_UBUNTU}
    IMPI2021_PATH=${IMPI2021_PATH_UBUNTU}
    MVAPICH2_PATH=${MVAPICH2_PATH_UBUNTU}
    MVAPICH2X_PATH=${MVAPICH2X_PATH_UBUNTU}
    OPENMPI_PATH=${OPENMPI_PATH_UBUNTU}
    CHECK_AOCL=0
    CHECK_NCCL=1
    CHECK_GCC=0
    CHECK_DOCKER=1
elif [[ $distro == "SUSE Linux Enterprise High Performance Computing 15 SP4" ]]
then
    # add /sbin and /usr/sbin to the path to allow lscpi and ibstatus called without path
    # as only UID=0 get it by default
    export PATH=$PATH:/sbin:/usr/sbin
    MKL_VERSION="2023.1.0"
    #
    CHECK_GCC=0
    CHECK_ONEAPI=1
    CHECK_AZURE_HPC_DIAG=1

    HPCX_OMB_PATH=${HPCX_OMB_PATH_SUSE}
    CHECK_HPCX=1
    CHECK_IMPI_2021=1
    CHECK_MVAPICH2=1
    CHECK_MVAPICH2X=0
    CHECK_CUDA=1
    CHECK_OMPI=1
    CHECK_BLIS_MT=1

    MODULE_FILES_ROOT=${MODULE_FILES_ROOT_SUSE}
    MOFED_VERSION=${MOFED_VERSION_SUSE}
    IMPI2021_PATH=${IMPI2021_PATH_SUSE}
    MVAPICH2_PATH=${MVAPICH2_PATH_SUSE}
    OPENMPI_PATH=${OPENMPI_PATH_SUSE}

    CHECK_AOCL=1
    CHECK_NV_PMEM=0
    CHECK_NCCL=1
else
    echo "*** Error - invalid distro!"
    exit -1
fi

module use ${MODULE_FILES_ROOT}

# check file is present
check_exists() {
    ls $1
    if [ $? -eq 0 ]
    then
        echo "$1 [OK]"
    else
        echo "*** Error - $1 not found!" >&2
        exit -1
    fi
}

# check exit code
check_exit_code() {
    if [ $? -eq 0 ]
    then
        echo "[OK] : $1"
    else
        echo "*** Error - $2!" >&2
        exit -1
    fi
}

# verify if package updates work
case ${distro} in
    Ubuntu*) sudo apt-get -q --assume-no update;;
    CentOS* | AlmaLinux*) sudo yum update -y --setopt tsflags=test;;
    * ) ;;
esac
check_exit_code "Package update works" "Package update fails!"

# verify MOFED installation
if [[ $distro == "SUSE Linux Enterprise High Performance Computing 15 SP4" ]]
then
  echo "ofed inbox driver does miss the ofed_info tool"
else
  ofed_info | grep ${MOFED_VERSION}
  check_exit_code "MOFED installed" "MOFED not installed"
fi

# verify IB device is listed
lspci | grep "Infiniband controller\|Network controller"
check_exit_code "IB device is listed" "IB device not found"

# verify IB device is up
ibstatus | grep "LinkUp"
check_exit_code "IB device state: LinkUp" "IB link not up"

# verify GCC modulefile
if [ $CHECK_GCC -eq 1 ]
then
    # Not using gcc 9.2.0 in Ubuntu 20.04 (9.3.0 used)
    check_exists "${MODULE_FILES_ROOT}/gcc-${GCC_VERSION}"
fi

# verify s/w package installations
if [ $CHECK_GCC -eq 1 ]
then
    # Not using gcc 9.2.0 in Ubuntu 20.04 (9.3.0 used)
    check_exists "/opt/gcc-${GCC_VERSION}/"
fi

check_exists "/opt/intel/oneapi/mkl/${MKL_VERSION}/"

# verify hpcdiag installation
check_exists '/opt/azurehpc/diagnostics/gather_azhpc_vm_diagnostics.sh'

if [ $CHECK_AOCL -eq 1 ]
then
    # verify AMD modulefiles
    check_exists "${MODULE_FILES_ROOT}/amd/aocl"

    check_exists "/opt/amd/lib/"
    check_exists "/opt/amd/include/"
fi

if [ $CHECK_DOCKER -eq 1 ] && [ "${MOFED_LTS}" = false ]
then
    sudo docker pull hello-world
    sudo docker run hello-world
    check_exit_code "Docker installed and working correctly!" "Problem with Docker!"
fi

# verify mpi installations and their modulefiles
module avail

# hpcx
if [ $CHECK_HPCX -eq 1 ]
then
    check_exists "${MODULE_FILES_ROOT}/mpi/hpcx"

    module load mpi/hpcx
    mpirun -np 2 --map-by ppr:2:node -x UCX_TLS=rc ${HPCX_OMB_PATH}/osu_latency
    check_exit_code "HPC-X" "Failed to run HPC-X"
    module unload mpi/hpcx
fi

# impi 2021
if [ $CHECK_IMPI_2021 -eq 1 ]
then
    check_exists "${MODULE_FILES_ROOT}/mpi/impi-2021"

    module load mpi/impi-2021
    mpiexec -np 2 -ppn 2 -env FI_PROVIDER=mlx -env I_MPI_SHM=0 ${IMPI2021_PATH}/bin/IMB-MPI1 pingpong
    check_exit_code "Intel MPI 2021" "Failed to run Intel MPI 2021"
    module unload mpi/impi-2021
fi

# impi 2018
if [ $CHECK_IMPI_2018 -eq 1 ]
then
    check_exists "${MODULE_FILES_ROOT}/mpi/impi"

    module load mpi/impi
    mpiexec -np 2 -ppn 2 -env I_MPI_FABRICS=ofa ${IMPI2018_PATH}/linux/mpi/intel64/bin/IMB-MPI1 pingpong
    check_exit_code "Intel MPI 2018" "Failed to run Intel MPI 2018"
    module unload mpi/impi
fi

# mvapich2
if [ $CHECK_MVAPICH2 -eq 1 ]
then
    check_exists "${MODULE_FILES_ROOT}/mpi/mvapich2"
    module load mpi/mvapich2

    # Env MV2_FORCE_HCA_TYPE=22 explicitly selects EDR
    mpiexec -np 2 -ppn 2 -env MV2_USE_SHARED_MEM=0  -env MV2_FORCE_HCA_TYPE=22  ${MVAPICH2_PATH}/osu-micro-benchmarks/mpi/pt2pt/osu_latency
    check_exit_code "MVAPICH2" "Failed to run MVAPICH2"
    module unload mpi/mvapich2
fi

# mvapich2x
if [ $CHECK_MVAPICH2X -eq 1 ]
then
    check_exists "${MODULE_FILES_ROOT}/mpi/mvapich2x"
    check_exists ${MVAPICH2X_PATH}

    module load mpi/mvapich2x
    mpiexec -np 2 -ppn 2 -env MV2_USE_SHARED_MEM=0  ${MVAPICH2X_INSTALLATION_DIRECTORY}/libexec/osu-micro-benchmarks/mpi/pt2pt/osu_latency
    check_exit_code "MVAPICH2X" "Failed to run MVAPICH2X"
    module unload mpi/mvapich2x
fi

# Note: no need to run OpenMPI, as it is already covered by HPC-X runs, but make sure it is installed
if [ $CHECK_OMPI -eq 1 ]
then
    check_exists "${MODULE_FILES_ROOT}/mpi/openmpi"
    check_exists ${OPENMPI_PATH}
fi

# Check Cuda drivers by running Nvidia SMI
if [ $CHECK_CUDA -eq 1 ]
then
    nvidia-smi
    check_exit_code "Nvidia SMI - Cuda Drivers" "Failed to run Nvidia SMI - Cuda Drivers"
fi

# Perform Single Node NCCL Test
if [ $CHECK_NCCL -eq 1 ]
then
    module load mpi/hpcx

    if [ "${MOFED_LTS}" = true ]
    then
        mpirun -np 4 \
        -x LD_LIBRARY_PATH \
        --allow-run-as-root \
        --map-by ppr:4:node \
        -mca coll_hcoll_enable 0 \
        -x UCX_TLS=tcp \
        -x CUDA_DEVICE_ORDER=PCI_BUS_ID \
        -x NCCL_SOCKET_IFNAME=eth0 \
        -x NCCL_DEBUG=WARN \
        /opt/nccl-tests/build/all_reduce_perf -b1K -f2 -g1 -e 4G
    else
        mpirun -np 8 \
        --allow-run-as-root \
        --map-by ppr:8:node \
        -x LD_LIBRARY_PATH=/usr/local/nccl-rdma-sharp-plugins/lib:$LD_LIBRARY_PATH \
        -mca coll_hcoll_enable 0 \
        -x UCX_TLS=tcp \
        -x CUDA_DEVICE_ORDER=PCI_BUS_ID \
        -x NCCL_SOCKET_IFNAME=eth0 \
        -x NCCL_DEBUG=WARN \
        -x NCCL_NET_GDR_LEVEL=5 \
        /opt/nccl-tests/build/all_reduce_perf -b1K -f2 -g1 -e 4G
    fi

    check_exit_code "Single Node NCCL Test" "Failed"

    module unload mpi/hpcx
fi

echo "ALL OK!"

exit 0


#!/bin/bash

##This is a test to verify an NDv4 vm is seeing the expected number of GPUs
##and NICs.

#Catch error codes that may be thrown by the executable passed as the first
#input, and if an error code is tripped throw the second input as a message
catch_error() {
	output=$($1)
	err_code=$?
	if [ $err_code -ne 0 ]; then
		echo "$2 $err_code" >&2
		exit $err_code;
	fi
}

#Store output from calls passed to catch_error
output=""

#Count the number of gpu-name nvidia-smi outputs.
gpu_names="timeout 3m nvidia-smi --query-gpu=name --format=csv,noheader"
error_smi="Fail: nvidia-smi failed with error code"
catch_error "$gpu_names" "$error_smi"
ngpus=$(echo "$output" | wc -l)

smi_warnings="timeout 3m nvidia-smi"
catch_error "$smi_warnings" "$error_smi"
out_warn=$(echo "$output")
nwarn=$(echo "$out_warn" | grep "WARNING" | wc -l)

#Count the number of nics lshw detects.
find_nics="sudo timeout 3m lshw -C network"
error_nics="Fail: lshw failed with error code"
catch_error "$find_nics" "$error_nics"
nnics=$(echo "$output" | grep -i ConnectX-6 | wc -l)

#Count the number of nvmedrives.
find_nvme="sudo timeout 3m lshw -C storage"
error_nvme="Fail: lshw failed with error code"
catch_error "$find_nvme" "$error_nvme"
nnvme=$(echo "$output" | grep nvme | grep logical | wc -l)

#Did either test fail?
passed=1
if [ $nwarn -ne 0 ]; then
	passed=0
	echo "nvidia-smi reported warnings, see output below:"
	echo "$out_warn"
fi

if [ $ngpus -ne 8 ]; then
	passed=0
	echo "$ngpus GPUs detected with 8 expected."
fi

if [ $nnics -ne 8 ]; then
	passed=0
	echo "$nnics NICs detected with 8 expected."
fi

if [ $nnvme -ne 8 ]; then
	passed=0
	echo "$nnvme NVMEs detected with 8 expected."
fi

if [ $passed -ne 1 ]; then
	echo "Fail: Error when detecting GPUs, NVMEs, and NICs."
	exit 1;
else
	echo "Correct number of GPUs, NVMEs, and NICs detected."
fi


#!/bin/bash

##This script runs dcgmi diag -r 1 tests and evaluates if any fail.

#Catch error codes that may be thrown by the executable passed as the first
#input, and if an error code is tripped throw the second input as a message.
catch_error() {
	output=$($1)
	err_code=$?
	if [ $err_code -ne 0 ]; then
		echo -e "\t $2 $err_code" >&2
		exit $err_code;
	fi
}

output=""
error_smi="**Fail** nvidia-smi failed with error code"

echo "dcgmi diag tests:"

#get the number of GPUs
exec_names="timeout 3m nvidia-smi --query-gpu=name --format=csv,noheader"
catch_error "$exec_names" "$error_smi"
ngpus=$(echo "$output" | wc -l)

#array to hold status of persistence mode for each gpu
declare -a pmvals=()

#Find which GPUs have persistence mode off, record that information in pmvals.
#Enable persistence mode on GPUs that have it disabled.
for i in $(seq 0 $((ngpus-1))); do
	exec_query="timeout 3m nvidia-smi -i $i -q"
	catch_error "$exec_query" "$error_smi"
	pmvals[$i]=$(echo "$output" | grep "Persistence Mode" \
		| awk '{print $4}')
	if [ ${pmvals[$i]} == "Disabled" ]; then
		exec_pm="timeout 3m sudo nvidia-smi -i $i -pm 1"
                catch_error "$exec_pm" "$error_smi"
	fi
done

#Turn on persistence mode.

#Run the basic (r=1) dcgmi diag tests.
exec_dcgmi="timeout 3m dcgmi diag -r 1"
error_dcgmi="**Fail** dcgmi diag failed with error code"
catch_error "$exec_dcgmi" "$error_dcgmi"
fails=$(echo "$output" | grep "Fail" | wc -l)

#Disable persistence mode on GPUs that started with it disabled.
for i in $(seq 0 $((ngpus-1))); do
	if [ ${pmvals[$i]} == "Disabled" ]; then
		exec_pm="timeout 3m sudo nvidia-smi -i $i -pm 0"
                catch_error "$exec_pm" "$error_smi"
	fi
done

if [ $fails -gt 0 ]; then
	echo -e " \t **Fail** dcgmi tests have hit errors. Execute the"\
		"command 'dcgmi diag -r 1' to see specific failures"
	exit 1
else
	echo -e " \t **Pass** All level 1 dcgmi diag tests passed."
fi


#!/bin/bash

#ECC errors are handled by row remapping on A100. This script first counts the
#row remappings due to ECC errors (correctable and otherwise). If any GPU has
#512 or  more, the test fails.

#Catch error codes that may be thrown by the executable passed as the first
#input, and if an error code is tripped throw the second input as a message
catch_error() {
	output=$($1)
	err_code=$?
	if [ $err_code -ne 0 ]; then
		echo -e "\t $2 $err_code" >&2
		exit $err_code;
	fi
}

#Store output from calls passed to catch_error
output=""

echo "ECC error check:"

pass=1

DRAM_ECC_THRESHOLD=20000000
SRAM_ECC_THRESHOLD=10000

#query the number of gpus
exec_names="timeout 3m nvidia-smi --query-gpu=name --format=csv,noheader"
error_smi="**Fail** nvidia-smi failed with error code"
catch_error "$exec_names" "$error_smi"
ngpus=$(echo "$output" | wc -l)

#query nvidia-smi to collect row remap information.
exec_rows="timeout 3m nvidia-smi -q -d ROW_REMAPPER"
catch_error "$exec_rows" "$error_smi"
row_remaps=$(echo "$output")

#query nvidia-smi to collect row remap information.
exec_ecc="timeout 3m nvidia-smi -q -d ECC"
catch_error "$exec_ecc" "$error_smi"
ECC_errors=$(echo "$output")

#Query the nvidia-smi output to identify any pending row remaps.
pending=$(echo "$row_remaps" | grep "Pending" | cut -d: -f2)
gpui=0

#Check if any GPU has a pending remap.
for pend in $pending; do
	if [ "$pend" != "No" ]; then
		echo "$gpui has a pending remap. Reset the GPU."
		exit
		pass=0
	fi
	gpui=$((gpui+1))
done

#Query the nvidia-smi output to identify any row remap failures.
fails=$(echo "$row_remaps" | grep "Remapping Failure Occurred" | cut -d: -f2)
gpui=0

#Check if any GPU has had a row remap failure.
for err in $fails; do
	if [ "$err" != "No" ]; then
		echo "GPU $gpui indicates it has experienced a remap"\
	       	"error."
		gpui=$((gpui+1))
		pass=0
	fi
done

#Query the nvidia-smi output to identify remap counts due to correctable
#and uncorrectable ECC errors.

correctable=$(echo "$row_remaps" | grep "Correctable Error" | cut -d: -f2)
uncorrectable=$(echo "$row_remaps" | grep "Uncorrectable Error" | cut -d: -f2)

correctable_v=( ${correctable} )
uncorrectable_v=( ${uncorrectable} )

#Identify any GPUs that have experienced more row remaps than allowed.
for i in $(seq 0 $((ngpus-1))); do
	val=$((uncorrectable_v[i]))
	if [ $val -gt 512 ]; then
		pass=0
		echo "GPU $i has $val uncorrectable row remap errors."
	fi
done

#Check the number of uncorrectable volatile SRAM ECC errors. Fail if any are
#observed
uncorrectableVS=$(echo "$ECC_errors" | grep Volatile -A 4 \
	| grep "SRAM Uncorrectable" | cut -d: -f2)
uncorrectableVS_v=( ${uncorrectableVS} )
for i in $(seq 0 $((ngpus-1))); do
	val=$((uncorrectableVS_v[i]))
	if [ $val -gt 0 ]; then
		pass=0
		echo "GPU $i has $val uncorrectable volatile SRAM errors."
	fi
done

#Count the number of correctable volatile SRAM ECC errors. Warn if more than
#SRAM_ECC_THRESHOLD are observed, but don't fail.
correctableVS=$(echo "$ECC_errors"  | grep Volatile -A 4 \
	| grep "SRAM Correctable" | cut -d: -f2)
correctableVS_v=( ${correctableVS} )
for i in $(seq 0 $((ngpus-1))); do
	val=$((correctableVS_v[i]))
	if [ $val -gt $SRAM_ECC_THRESHOLD ]; then
		echo "GPU $i has $val correctable volatile SRAM errors."
	fi
done

#Count the number of uncorrectable volatile DRAM ECC errors. Warn if more than
#DRAM_ECC_THRESHOLD are observed but don't fail.
uncorrectableVD=$(echo "$ECC_errors" | grep Volatile -A 4 \
	| grep "DRAM Uncorrectable" | cut -d: -f2)
uncorrectableVD_v=( ${uncorrectableVD} )
for i in $(seq 0 $((ngpus-1))); do
	val=$((uncorrectableVD_v[i]))
	if [ $val -gt $DRAM_ECC_THRESHOLD ]; then
		echo "GPU $i has $val uncorrectable volatile DRAM errors."
	fi
done

#Count the number of correctable volatile DRAM ECC errors. Warn if more than
#DRAM_ECC_THRESHOLD are observed but don't fail.
correctableVD=$(echo "$ECC_errors" | grep Volatile -A 4 \
	| grep "DRAM Correctable" | cut -d: -f2)
correctableVD_v=( ${correctableVD} )
for i in $(seq 0 $((ngpus-1))); do
	val=$((correctableVD_v[i]))
	if [ $val -gt $DRAM_ECC_THRESHOLD ]; then
		echo "GPU $i has $val correctable volatile DRAM errors."
	fi
done

#Check the number of uncorrectable aggregate SRAM ECC errors. Fail if any are
#detected.
uncorrectableAS=$(echo "$ECC_errors" | grep Aggregate -A 4 \
	| grep "SRAM Uncorrectable" | cut -d: -f2)
uncorrectableAS_v=( ${uncorrectableAS} )
for i in $(seq 0 $((ngpus-1))); do
	val=$((uncorrectableAS_v[i]))
	if [ $val -gt 0 ]; then
		pass=0
		echo "GPU $i has $val uncorrectable aggregate SRAM errors."
	fi
done

#Count the number of correctable aggregate SRAM ECC errors. Warn if more than
#SRAM_ECC_THRESHOLD are detected but don't fail.
correctableAS=$(echo "$ECC_errors" | grep Aggregate -A 4 \
	| grep "SRAM Correctable" | cut -d: -f2)
correctableAS_v=( ${correctableAS} )
for i in $(seq 0 $((ngpus-1))); do
	val=$((correctableAS_v[i]))
	if [ $val -gt $SRAM_ECC_THRESHOLD ]; then
		echo "GPU $i has $val correctable aggregate SRAM errors."
	fi
done

#Count the number of uncorrectable aggregate DRAM ECC errors. Warn if more than
#DRAM_ECC_THRESHOLD are detected but don't fail.
uncorrectableAD=$(echo "$ECC_errors" | grep Aggregate -A 4 \
	| grep "DRAM Uncorrectable" | cut -d: -f2)
uncorrectableAD_v=( ${uncorrectableAD} )
for i in $(seq 0 $((ngpus-1))); do
	val=$((uncorrectableAD_v[i]))
	if [ $val -gt $DRAM_ECC_THRESHOLD ]; then
		echo "GPU $i has $val uncorrectable aggregate DRAM errors."
	fi
done

#Count the number of correctable aggregate DRAM ECC errors. Warn if more than
#DRAM_ECC_THRESHOLD are detected but don't fail.
correctableAD=$(echo "$ECC_errors" | grep Aggregate -A 4 \
	| grep "DRAM Correctable" | cut -d: -f2)
correctableAD_v=( ${correctableAD} )
for i in $(seq 0 $((ngpus-1))); do
	val=$((correctableAD_v[i]))
	if [ $val -gt $DRAM_ECC_THRESHOLD ]; then
		echo "GPU $i has $val correctable aggregate DRAM errors."
	fi
done

if [ $pass -ne 1 ]; then
	echo -e "\t **Fail** According to nvidia-smi at least one GPU had an"\
		"ECC error status to investigate."
	exit 1
else
	echo -e "\t **Pass** The ECC error check passed."
fi


#!/bin/bash

#Users may pass a directory where health checks are located to this script.
#If no directory is passed the script assumes health checks are in the opt
#directory.

#Verify if an argument was passed to the script. Test if the argument is
#a directory that contains all the expected health checks. If no arguments
#are passed use the default directory.
declare -a files=("check-hardware.sh" "dcgmi-check.sh" "ecc-errors.sh"\
	   "test-bandwidth.sh" "test-ib.sh" "test-nccl.sh"\
	   "test-MaxPayLoad.sh" "gpu-copy.cu")
if [ "$#" -gt 1 ]; then
	echo "Usage: run-health-checks.sh [DIRECTORY]"
	echo "DIRECTORY is an optional argument that specifies the location of"
	echo "the test scripts to run. It defaults to the install location of"
	echo "this script."
	exit 1;
fi
if [ "$#" -eq 0 ]; then
	pass=1
	DIR=$(cd `dirname $0` && pwd)
	for file in "${files[@]}"; do
		if ! test -f "$DIR/$file"; then
			pass=0
		fi
	done
	#if tests are located in script dir use those
	if [[ $pass -eq 1 ]]; then
		TEST_DIR=$DIR
	fi
	if [[ $pass -eq 0 ]]; then
		DIR=$PWD
		pass=1
		for file in "${files[@]}"; do
			if ! test -f "$DIR/$file"; then
				pass=0
			fi
		done
		if [[ $pass -eq 1 ]]; then
			TEST_DIR=$DIR
		fi
	fi
	if [[ $pass -eq 0 ]]; then
		echo "Not all healch check files found in the working"\
			"directory please specify where they can be found."
		exit 1;
	fi
else
	pass=1
	for file in "${files[@]}"; do
		if ! test -f "$1/$file"; then
			echo "Directory $1 is missing $file"
			pass=0
		fi
	done
	if [ $pass -eq 1 ]; then
		TEST_DIR=$1
	else
		echo "Directory $1 is missing health check files."
		exit 1;
	fi

fi

check_hardware=$($TEST_DIR/check-hardware.sh)
detect=$?
#detect=$(echo "$check_hardware" | grep "Fail:" | wc -l)
echo "------------------------------------------------------------------------"
echo "**************************Begin Health Checks***************************"
echo "------------------------------------------------------------------------"
line="------------------------------------------------------------------------"
if [ $detect -ne 0 ]; then
	echo "Failed to detect GPUs and NICs with error code $detect"\
		", run check-hardware.sh"
else
	pass=1
        #run 'dcgmi diag -r 1' and check for failures
	$TEST_DIR/dcgmi-check.sh
	out=$?
	if [ $out -ne 0 ]; then
		pass=0
	fi
        echo "$line"

        #run 'nvidia-smi -q -d ROW_REMAPPER' and check for problems
	$TEST_DIR/ecc-errors.sh
	out=$?
	if [ $out -ne 0 ]; then
		pass=0
	fi
        echo "$line"

        #run ibstatus and check all devices are set up as expected
	$TEST_DIR/test-ib.sh
	out=$?
	if [ $out -ne 0 ]; then
		pass=0
	fi
        echo "$line"

        #test that host to device/device to host bandwidth meet expectations.
	$TEST_DIR/test-bandwidth.sh $TEST_DIR
	out=$?
	if [ $out -ne 0 ]; then
		pass=0
	fi
        echo "$line"

        #test nccl using nvlink and ib
	$TEST_DIR/test-nccl.sh
	out=$?
	if [ $out -ne 0 ]; then
		pass=0
	fi
        echo "$line"

	$TEST_DIR/test-MaxPayLoad.sh
	out=$?
	if [ $out -ne 0 ]; then
		pass=0
	fi
        echo "$line"

	if [ $pass -ne 1 ]; then
                message="******************At least one health check failed."
                message+="*********************"
                echo "$message"
	else
                message="********************All health checks have passed."
                message+="**********************"
                echo "$message"
	fi
        echo "$line"
fi


#!/bin/bash

##This script tests host to device bandwidth and device to host bandwidth using
##a script recorded within the test that tests host to device and device to
##host copy bandwidths between every combination of gpus and numa domains. The
##test confirms that each GPU is hitting the expected bandwidths to at least
##one of the numa domains.
##
##When this test is first run it will compile the gpu-copy benchmark if it is
##not detected.

#Catch error codes that may be thrown by the executable passed as the first
#input, and if an error code is tripped throw the second input as a message
catch_error() {
	output=$($1)
	err_code=$?
	if [ $err_code -ne 0 ]; then
		echo -e "\t $2 $err_code" >&2
		exit $err_code;
	fi
}

create_benchmark() {
        file_dir="$1"

        compile_bench="nvcc -lnuma $file_dir/gpu-copy.cu -o $file_dir/gpu-copy"
	compile_error="**Fail** Error when attempting to compile benchmark"
        catch_error "$compile_bench" "$compile_error"
}

locate_file() {
        file="$1"
        exitOnFail=$2
        pass=1
	#File name we want to locate.
        local file="$1"
	#What should  happen if the file isn't located?
        local exitOnFail=$2
	#Write the directory containing the file here, if found.
        local return_dir=$3
        local pass=1

	#Start by testing directory where the script is located
        local DIR=$(cd `dirname $0` && pwd)
        DIR=$(cd `dirname $0` && pwd)
        if ! test -f "$DIR/$file"; then
                pass=0
        fi
        #If file is located in script dir use that directory.
        if [[ $pass -eq 1 ]]; then
                CHECK_LOC=$DIR
        fi
	#Otherwise check the PWD (in case that is different).
        if [[ $pass -eq 0 ]]; then
                DIR=$PWD
                pass=1
                if test -f "$DIR/$file"; then
                        CHECK_LOC=$DIR
                else
                        pass=0
                fi
        fi

	#If we found the file pass that to return_dir.
	#Otherwise either just exit or also fail.
        if [[ $exitOnFail -eq 0 ]]; then
                if [[ $pass -eq 0 ]]; then
                         DIR=$(cd `dirname $0` && pwd)
                         CHECK_LOC=$DIR
                fi
        else
                if [[ $pass -eq 0 ]]; then
                        echo "Cannot find file $file."
                        exit 1;
                fi
        fi
	eval $return_dir="'$CHECK_LOC'"
}

echo "htod and dtoh bandwidth check:"

#If no arguments are passed to the script assume the gpu-copy test is located
#in the default location. Otherwise check if it the gpu-copy test is in the
#location indicated by the argument.
if [ "$#" -eq 0 ]; then
	locate_file "gpu-copy" 0 CHECK_DIR
	locate_file "gpu-copy.cu" 1 CHECK_DIR
        create_benchmark "$CHECK_DIR"
else
	pass=1
	if test -f "$1/gpu-copy"; then
		CHECK_DIR=$1
	else
                create_benchmark $1
                CHECK_DIR=$1
	fi
fi
#Store output from calls passed to catch_error
output=""

#Count the number of gpu-name nvidia-smi outputs.
error_smi="**Fail** nvidia-smi failed with error code"
#Lock graphics clocks to 1400 to eliminate any time for the GPUs to boost.
#This likely isn't important for performance here, but we will do it anyway
#to be safe.
lock_clocks="sudo nvidia-smi -lgc 1400"
catch_error "$lock_clocks" "$error_smi"

#Count the GPUs.
gpu_list="timeout 3m nvidia-smi --query-gpu=name --format=csv,noheader"
catch_error "$gpu_list" "$error_smi"
ngpus=$(echo "$output" | wc -l)

#Run the superbench device to host bandwidth test.
exec_htod="timeout 3m $CHECK_DIR/gpu-copy --size 134217728 --htod"

error_htod="**Fail** The htod gpu_copy test failed to execute."
error_htod+="It exited with error code"

catch_error "$exec_htod" "$error_htod"
x_htod=$(echo "$output")

#Run the superbench host to device bandwidth test.
exec_dtoh="timeout 3m $CHECK_DIR/gpu-copy --size 134217728 --dtoh"

error_dtoh="**Fail** The dtoh gpu_copy test failed to execute."
error_dtoh+="It exited with error code"
catch_error "$exec_dtoh" "$error_dtoh"
x_dtoh=$(echo "$output")

pass=1

#Loop over all of the detected GPUs.
for i in $(seq 0 $((ngpus-1))); do
	#Collect host to device bandwidths computed in each numa zone.
	bw_htod=$(echo "$x_htod" | grep "gpu$i" | cut -d' ' -f2 | cut -d. -f1)
	max_htodbw=0
	min_bw=100
	#Loop over the bandwidths observed in each numa zone and find max.
	for bw in $bw_htod; do
		if [ $max_htodbw -lt $bw ]; then
			max_htodbw=$bw
		fi
	done

	#Collect device to host bandwidths computed in each numa zone.
	bw_dtoh=$(echo "$x_dtoh" | grep "gpu$i" | cut -d' ' -f2 | cut -d. -f1)
	max_dtohbw=0
	#Loop over bandwidths observed in each numa zone and find max.
	for bw in $bw_dtoh; do
		if [ $max_dtohbw -lt $bw ]; then
			max_dtohbw=$bw
		fi
	done
	#Find minimum of the htod and dtoh bandwidths.
	if [ $max_htodbw -lt $max_dtohbw ]; then
		min_bw=$max_htodbw
	else
		min_bw=$max_dtohbw
	fi

	#If the min bandwidth is too low the test has failed.
	if [ $min_bw -lt 24 ]; then
		echo "Bandwidth is low on device $i. Reported bandwidth is"\
			"$min_bw GB/s."
		pass=0
	fi
done
#Unlock the graphics clock.
unlock_clocks="sudo timeout 3m nvidia-smi -rgc"
catch_error "$unlock_clocks" "$error_smi"

if [ $pass -ne 1 ]; then
	echo -e "\t **Fail** At least one device reported low htod or dtoh"\
		"bandwidth."
	exit 1;
else
	echo -e "\t **Pass** The htod and dtoh bandwidth checks both passed"
fi


#!/bin/bash

##This script examines the output of ibstatus, looking for any problems.

#Catch error codes that may be thrown by the executable passed as the first
#input, and if an error code is tripped throw the second input as a message
catch_error() {
	output=$($1)
	err_code=$?
	if [ $err_code -ne 0 ]; then
		echo -e "\t $2 $err_code" >&2
		exit $err_code;
	fi
}
echo "check ibstatus:"

#Store output from calls passed to catch_error
output=""

ib=200;
num_r=0
num_s=0
num_ps=0

#Check ibstatus Rates and make sure they are all 200.
exec_ibstat="timeout 3m ibstatus"
error_ibstat="**Fail** ibstatus exited with error code"
catch_error "$exec_ibstat" "$error_ibstat"
ibstat_out=$(echo "$output")

#make sure we are only checking rates for InfiniBand
types=$(echo "$ibstat_out" | grep "link_layer:" | awk '{print $2}')
types_v=( ${types} )

for i in $(echo "$ibstat_out" | grep "rate:" | awk '{print $2}'); do
	num_r=$((num_r+1));
	if [ $i -lt $ib ]; then
		ir=$((num_r-1))
		if [ "${types_v[ir]}" = "InfiniBand" ]; then
			ib=$i;
		fi
	fi;
done

#Check ibstatus to make sure States are set to Active.
for i in $(echo "$ibstat_out" | grep "state:" | grep -v "phys" | awk '{print $3}'); do
	if [ "$i" = "ACTIVE" ]; then
		num_s=$((num_s+1));
	fi
done

#Check ibstatus to make sure Physical states are set to LinkUp
for i in $(echo "$ibstat_out" | grep "phys state:" | awk '{print $4}'); do
	if [ "$i" = "LinkUp" ]; then
		num_ps=$((num_ps+1));
	fi
done

pass=1

if [ $ib -lt 200 ]; then
	echo "The minimum ib rate observed is $ib, but it should be 200."
	pass=0
fi

if [ $num_s -lt $num_r ]; then
	echo "The ib 'State' is not set to 'ACTIVE' for all devices."
	pass=0
fi

if [ $num_ps -lt $num_r ]; then
	echo "The ib 'Physical state' is not set to LinkUp for all devices."
	pass=0
fi

if [ $pass -eq 1 ]; then
	echo -e "\t **Pass** All ibstatus settings are ok."
else
	echo -e "\t **Fail** The settings returned by ibstatus indicate a problem."
	exit 1;
fi


#!/bin/bash

echo "MaxPayload test:"

pass=1
ids=$(lspci | grep "3D controller" | cut -d' ' -f1)
for i in $(echo "$ids"); do
	DevCap=$(sudo lspci -vv -s $i | grep DevCap: \
		| cut -d' ' -f2)
	DevCtl=$(sudo lspci -vv -s $i | grep DevCtl: -A 2 \
		| grep MaxPayload | cut -d' ' -f2)
	if [[ $DevCap -ne $DevCtl ]]; then
		echo "DevCap and DevCtl MaxPayload do not match for device $i"
		exit 1;
	fi
done
echo -e "\t **Pass** The MaxPayload size check passed."


#!/bin/bash

##This script executes two nccl tests, one using ib and one using nvlink.
##It also checks that the performance on these benchmarks is satisfactory.

#Catch error codes that may be thrown by the executable passed as the first
#input, and if an error code is tripped throw the second input as a message
catch_error() {
	output=$($1)
	err_code=$?
	if [ $err_code -ne 0 ]; then
		echo -e "\t $2 $err_code" >&2
		exit $err_code;
	fi
}
echo "NCCL tests:"

#Store output from calls passed to catch_error
output=""

pass=1

#Count the number of gpu-name nvidia-smi outputs.
error_smi="**Fail** nvidia-smi failed with error code"
#Load the mpi module.
module load mpi/hpcx > /dev/null

#Lock graphics clocks to 1400.
lock_clocks="sudo nvidia-smi -lgc 1400"
catch_error "$lock_clocks" "$error_smi"

mpi_text="mpirun -np 8 --bind-to numa --map-by ppr:8:node -x LD_LIBRARY_PATH="
mpi_text+="/usr/local/nccl-rdma-sharp-plugins/lib:\$LD_LIBRARY_PATH"
mpi_text+=" -mca coll_hcoll_enable 0 -x NCCL_IB_PCI_RELAXED_ORDERING=1"

exec_text="/opt/nccl-tests/build/all_reduce_perf -c 1 -b4G -f2 -g1 -e 4G"

exec_nvlink="timeout 3m $mpi_text -x UCX_TLS=tcp -x UCX_NET_DEVICES=eth0"
exec_nvlink+=" -x CUDA_DEVICE_ORDER=PCI_BUS_ID -x NCCL_SOCKET_IFNAME=eth0 -x"
exec_nvlink+=" NCCL_DEBUG=WARN -x NCCL_TOPO_FILE=/opt/microsoft/ndv4-topo.xml"
exec_nvlink+=" $exec_text"

error_nvlink="**Fail** nccl test using nvlink failed with error code"

#Run a single node nccl test using nvlink.
catch_error "$exec_nvlink" "$error_nvlink"
nccl_nvlink=$(echo "$output")

#select the average bandwidth from the nccl output.
bw=$(echo "$nccl_nvlink" | grep "Avg bus bandwidth" | awk '{print $6}')
#the average bandwidth should be above 235 GB/s.
less=$(echo "scale=4; 230.0 > $bw" | bc)
if [ $less -eq 1 ]; then
       echo -e "\t **Fail** observed bandwidth in nvlink nccl test is $bw."
       pass=0
fi

#Run a single node nccl test using infiniBand.
exec_ib="timeout 3m $mpi_text -H localhost:8 -x UCX_IB_PCI_RELAXED_ORDERING=on"
exec_ib+=" -x UCX_TLS=tcp -x UCX_NET_DEVICES=eth0 -x"
exec_ib+=" CUDA_DEVICE_ORDER=PCI_BUS_ID -x NCCL_SOCKET_IFNAME=eth0 -x"
exec_ib+=" NCCL_NET_GDR_LEVEL=5 -x NCCL_TOPO_FILE=/opt/microsoft/ndv4-topo.xml"
exec_ib+=" -x NCCL_SHM_DISABLE=1 -x NCCL_P2P_DISABLE=1 $exec_text"

error_ib="**Fail** the nccl ib test exited with error code"

catch_error "$exec_ib" "$error_ib"
nccl_ib=$(echo "$output")

#select the average bandwidth from the nccl output.
bw=$(echo "$nccl_ib" | grep "Avg bus bandwidth" | awk '{print $6}')

#the average bandwidth should be above 20 GB/s.
less=$(echo "scale=4; 20.0 > $bw" | bc)
if [ $less -eq 1 ]; then
       echo -e "\t **Fail** observed bandwidth in ib nccl test is $bw."
       pass=0
fi

#Unlock the graphics clock.
unlock_clocks="sudo timeout 3m nvidia-smi -rgc"
catch_error "$unlock_clocks" "$error_smi"

if [ $pass -eq 1 ]; then
	echo -e "\t **Pass** Both nccl tests pass."
else
	exit 1;
fi


#!/bin/bash
set -e

KERNEL_VERSION=$(uname -r) su -c 'echo linux-image-$KERNEL_VERSION hold | dpkg --set-selections'
sed -i 's/APT::Periodic::Unattended-Upgrade ".*/APT::Periodic::Unattended-Upgrade "0";/' /etc/apt/apt.conf.d/20auto-upgrades

systemctl stop unattended-upgrades.service
systemctl disable unattended-upgrades.service
systemctl mask unattended-upgrades.service


#!/bin/bash
set -e

# Kernel Parameter to disable predictive network interface naming
KERNEL_PARAMETER="net.ifnames=0"
GRUB_CMDLINE_LINUX="$GRUB_CMDLINE_LINUX console=tty1 console=ttyS0 earlyprintk=ttyS0"

# Update 50-cloudimg-settings.cfg with additional kernel parameter
sed -i "s/${GRUB_CMDLINE_LINUX}/${GRUB_CMDLINE_LINUX} ${KERNEL_PARAMETER}/" /etc/default/grub.d/50-cloudimg-settings.cfg
# Generate grub file with updated parameters
update-grub


#!/bin/bash
set -ex

# Install Dependencies
pip3 install -U netifaces
pip3 install -U PyYAML

# Disable some unneeded services by default (administrators can re-enable if desired)
systemctl disable ufw

# Disable cloud-init
echo network: {config: disabled} | sudo tee /etc/cloud/cloud.cfg.d/99-disable-network-config.cfg
bash -c "cat > /etc/netplan/50-cloud-init.yaml" <<'EOF'
network:
    ethernets:
        eth0:
            dhcp4: true
    version: 2
EOF
netplan apply

# Update memory limits
cat << EOF >> /etc/security/limits.conf
*               hard    memlock         unlimited
*               soft    memlock         unlimited
*               hard    nofile          65535
*               soft    nofile          65535
*               hard    stack           unlimited
*               soft    stack           unlimited
EOF

echo "vm.zone_reclaim_mode = 1" >> /etc/sysctl.conf
echo "net.ipv4.neigh.default.gc_thresh1 = 4096" >> /etc/sysctl.conf
echo "net.ipv4.neigh.default.gc_thresh2 = 8192" >> /etc/sysctl.conf
echo "net.ipv4.neigh.default.gc_thresh3 = 16384" >> /etc/sysctl.conf
echo "sunrpc.tcp_max_slot_table_entries = 128" >> /etc/sysctl.conf

## Systemd service for starting sunrpc and adding setting parameters
cat <<EOF >/usr/sbin/sunrpc_tcp_settings.sh
#!/bin/bash

modprobe sunrpc
sysctl -p
EOF
chmod 755 /usr/sbin/sunrpc_tcp_settings.sh

cat <<EOF >/etc/systemd/system/sunrpc_tcp_settings.service
[Unit]
Description=Set sunrpc tcp settings

[Service]
User=root
Type=oneshot
ExecStart=/usr/sbin/sunrpc_tcp_settings.sh
RemainAfterExit=true

[Install]
WantedBy=multi-user.target
EOF

systemctl enable sunrpc_tcp_settings
systemctl start sunrpc_tcp_settings
systemctl is-active --quiet sunrpc_tcp_settings

error_code=$?
if [ ${error_code} -ne 0 ]
then
    echo "sunrpc_tcp_settings service inactive!"
    exit ${error_code}
fi

# Install WALinuxAgent
apt-get install -y python3-setuptools
pip3 install distro
WAAGENT_VERSION=2.9.0.4
$COMMON_DIR/write_component_version.sh "WAAGENT" ${WAAGENT_VERSION}
DOWNLOAD_URL=https://github.com/Azure/WALinuxAgent/archive/refs/tags/v${WAAGENT_VERSION}.tar.gz
wget ${DOWNLOAD_URL}
tar -xvf $(basename ${DOWNLOAD_URL})
pushd WALinuxAgent-${WAAGENT_VERSION}/
python3 setup.py install --register-service
popd

# Configure WALinuxAgent
sed -i -e 's/# OS.EnableRDMA=y/OS.EnableRDMA=y/g' /etc/waagent.conf
sed -i -e 's/Provisioning.MonitorHostName=n/Provisioning.MonitorHostName=y/g' /etc/waagent.conf
echo "Extensions.GoalStatePeriod=300" | sudo tee -a /etc/waagent.conf
echo "Extensions.InitialGoalStatePeriod=6" | sudo tee -a /etc/waagent.conf
echo "OS.EnableFirewallPeriod=300" | sudo tee -a /etc/waagent.conf
echo "OS.RemovePersistentNetRulesPeriod=300" | sudo tee -a /etc/waagent.conf
echo "OS.RootDeviceScsiTimeoutPeriod=300" | sudo tee -a /etc/waagent.conf
echo "OS.MonitorDhcpClientRestartPeriod=60" | sudo tee -a /etc/waagent.conf
echo "Provisioning.MonitorHostNamePeriod=60" | sudo tee -a /etc/waagent.conf
systemctl daemon-reload
systemctl restart walinuxagent

# Setting Linux NFS read-ahead limits
# Reference:
#    https://learn.microsoft.com/en-us/azure/azure-netapp-files/performance-linux-nfs-read-ahead
#    https://learn.microsoft.com/en-us/azure/storage/blobs/secure-file-transfer-protocol-support-how-to?tabs=azure-portal
cat > /etc/udev/rules.d/90-nfs-readahead.rules <<EOM
SUBSYSTEM=="bdi",
ACTION=="add",
PROGRAM="/usr/bin/awk -v bdi=$kernel 'BEGIN{ret=1} {if ($4 == bdi) {ret=0}} END{exit ret}' /proc/fs/nfsfs/volumes",
ATTR{read_ahead_kb}="15380"
EOM

udevadm control --reload


#!/bin/bash

AOCC_VERSION=4.0.0_1

# install dependency
wget https://download.amd.com/developer/eula/aocc-compiler/aocc-compiler-${AOCC_VERSION}_amd64.deb
apt install -y ./aocc-compiler-${AOCC_VERSION}_amd64.deb

rm aocc-compiler-${AOCC_VERSION}_amd64.deb

$COMMON_DIR/write_component_version.sh "AOCC" ${AOCC_VERSION}


#!/bin/bash
set -ex

# Install DCGM
# Reference: https://developer.nvidia.com/dcgm#Downloads
# the repo is already added during nvidia/ cuda installations
DCGM_VERSION=3.1.8
apt-get install -y datacenter-gpu-manager=1:${DCGM_VERSION}
$COMMON_DIR/write_component_version.sh "DCGM" ${DCGM_VERSION}

# Enable the dcgm service
systemctl --now enable nvidia-dcgm
systemctl start nvidia-dcgm
# Check if the service is active
systemctl is-active --quiet nvidia-dcgm
error_code=$?
if [ ${error_code} -ne 0 ]
then
    echo "DCGM is inactive!"
    exit ${error_code}
fi


#!/bin/bash
set -ex

# Install Moby Engine and CLI
apt-get install -y moby-engine

# Install NVIDIA Docker
# Reference: https://docs.nvidia.com/datacenter/cloud-native/container-toolkit/install-guide.html
# Setting up NVIDIA Container Toolkit
curl -s -L https://nvidia.github.io/nvidia-docker/gpgkey | apt-key add - \
&& curl -s -L https://nvidia.github.io/nvidia-docker/${DISTRIBUTION}/nvidia-docker.list | tee /etc/apt/sources.list.d/nvidia-docker.list
# MIG Capability on A100
curl -s -L https://nvidia.github.io/nvidia-container-runtime/experimental/${DISTRIBUTION}/nvidia-container-runtime.list | tee /etc/apt/sources.list.d/nvidia-container-runtime.list

apt-get update
# Install nvidia-docker package
# Install NVIDIA container toolkit and mark NVIDIA packages on hold
apt-get install -y nvidia-container-toolkit
apt-mark hold nvidia-container-toolkit
apt-mark hold libnvidia-container-tools
apt-mark hold libnvidia-container1

# Install NVIDIA container runtime and mark NVIDIA packages on hold
apt-get install -y nvidia-container-runtime
apt-mark hold nvidia-container-runtime

wget https://raw.githubusercontent.com/NVIDIA/nvidia-docker/master/nvidia-docker
cp nvidia-docker /bin/
chmod +x /bin/nvidia-docker
wget https://raw.githubusercontent.com/NVIDIA/nvidia-docker/master/daemon.json
cp daemon.json /etc/docker/

# Working setup can be tested by running a base CUDA container
# nvidia-docker run -e NVIDIA_VISIBLE_DEVICES=all nvidia/cuda:11.0-base nvidia-smi

# disabling aufs, btrfs, zfs and devmapper snapshotter plugins
mkdir -p /etc/containerd
cat << EOF | tee -a /etc/containerd/config.toml
disabled_plugins = ["cri", "zfs", "aufs", "btrfs", "devmapper"]
EOF

# restart containerd service
systemctl restart containerd

# status of containerd snapshotter plugins
ctr plugin ls

# enable and restart the docker daemon to complete the installation
systemctl enable docker
systemctl restart docker

# Write the docker version to components file
moby_version=$(apt list --installed | grep moby-engine | awk -F' ' '{print $2}')
docker_version=$(nvidia-docker --version | awk -F' ' '{print $3}')
$COMMON_DIR/write_component_version.sh "MOBY-ENGINE" ${moby_version}
$COMMON_DIR/write_component_version.sh "NVIDIA-DOCKER" ${docker_version::-1}

# Remove unwanted repos
rm -f /etc/apt/sources.list.d/nvidia*


#!/bin/bash
set -ex

# IntelÂ® oneAPI Math Kernel Library
case ${DISTRIBUTION} in
    "ubuntu18.04") INTEL_MKL_VERSION="2023.1.0.46342";
        RELEASE_VERSION="cd17b7fe-500e-4305-a89b-bd5b42bfd9f8";
        CHECKSUM="cc28c94cab23c185520b93c5a04f3979d8da6b4c90cee8c0681dd89819d76167";;
    "ubuntu20.04") INTEL_MKL_VERSION="2023.1.0.46342";
        RELEASE_VERSION="cd17b7fe-500e-4305-a89b-bd5b42bfd9f8";
        CHECKSUM="cc28c94cab23c185520b93c5a04f3979d8da6b4c90cee8c0681dd89819d76167";;
    "ubuntu22.04") INTEL_MKL_VERSION="2023.1.0.46342";
        RELEASE_VERSION="cd17b7fe-500e-4305-a89b-bd5b42bfd9f8";
        CHECKSUM="cc28c94cab23c185520b93c5a04f3979d8da6b4c90cee8c0681dd89819d76167";;
    *) ;;
esac

ONE_MKL_DOWNLOAD_URL=https://registrationcenter-download.intel.com/akdlm/IRC_NAS/${RELEASE_VERSION}/l_onemkl_p_${INTEL_MKL_VERSION}_offline.sh
$COMMON_DIR/write_component_version.sh "INTEL_ONE_MKL" ${INTEL_MKL_VERSION}
$COMMON_DIR/download_and_verify.sh ${ONE_MKL_DOWNLOAD_URL} ${CHECKSUM}
sh ./l_onemkl_p_${INTEL_MKL_VERSION}_offline.sh -s -a -s --eula accept


#!/bin/bash
set -ex

LUSTRE_VERSION=2.15.1-29-gbae0abe

source $UBUNTU_COMMON_DIR/setup_lustre_repo.sh

apt-get update
apt-get install -y amlfs-lustre-client-${LUSTRE_VERSION}=$(uname -r)

$COMMON_DIR/write_component_version.sh "LUSTRE" ${LUSTRE_VERSION}


#!/bin/bash

set -e

MONEO_VERSION=v0.3.1

# Dependencies
python3 -m pip install --upgrade pip

MONITOR_DIR=/opt/azurehpc/tools

mkdir -p $MONITOR_DIR

pushd $MONITOR_DIR

    git clone https://github.com/Azure/Moneo  --branch $MONEO_VERSION

    chmod 777 Moneo

    pushd Moneo/linux_service
        ./configure_service.sh $MONITOR_DIR/Moneo
    popd
popd

# add an slias for Moneo
if ! grep -qxF "alias moneo='python3 /opt/azurehpc/tools/Moneo/moneo.py'" /etc/bash.bashrc; then
    echo "alias moneo='python3 /opt/azurehpc/tools/Moneo/moneo.py'" >> /etc/bash.bashrc
fi

$COMMON_DIR/write_component_version.sh "MONEO" ${MONEO_VERSION}


#!/bin/bash
set -ex

# Parameters
HPCX_CHECKSUM=$1

# Load gcc
set CC=/usr/bin/gcc
set GCC=/usr/bin/gcc

INSTALL_PREFIX=/opt

# HPC-X v2.15
HPCX_VERSION="v2.15"
TARBALL="hpcx-${HPCX_VERSION}-gcc-MLNX_OFED_LINUX-5-$DISTRIBUTION-cuda12-gdrcopy2-nccl2.17-x86_64.tbz"
HPCX_DOWNLOAD_URL=https://content.mellanox.com/hpc/hpc-x/${HPCX_VERSION}/${TARBALL}
HPCX_FOLDER=$(basename ${HPCX_DOWNLOAD_URL} .tbz)

$COMMON_DIR/download_and_verify.sh ${HPCX_DOWNLOAD_URL} ${HPCX_CHECKSUM}
tar -xvf ${TARBALL}
mv ${HPCX_FOLDER} ${INSTALL_PREFIX}
HPCX_PATH=${INSTALL_PREFIX}/${HPCX_FOLDER}
$COMMON_DIR/write_component_version.sh "HPCX" $HPCX_VERSION

# MVAPICH2 2.3.7-1
MV2_VERSION="2.3.7-1"
MV2_DOWNLOAD_URL=http://mvapich.cse.ohio-state.edu/download/mvapich/mv2/mvapich2-${MV2_VERSION}.tar.gz
$COMMON_DIR/download_and_verify.sh $MV2_DOWNLOAD_URL "fdd971cf36d6476d007b5d63d19414546ca8a2937b66886f24a1d9ca154634e4"
tar -xvf mvapich2-${MV2_VERSION}.tar.gz
cd mvapich2-${MV2_VERSION}
# Error exclusive to Ubuntu 22.04
# configure: error: The Fortran compiler gfortran will not compile files that call
# the same routine with arguments of different types.
./configure $(if [[ ${DISTRIBUTION} == "ubuntu22.04" ]]; then echo "FFLAGS=-fallow-argument-mismatch"; fi) --prefix=${INSTALL_PREFIX}/mvapich2-${MV2_VERSION} --enable-g=none --enable-fast=yes && make -j$(nproc) && make install
cd ..
$COMMON_DIR/write_component_version.sh "MVAPICH2" ${MV2_VERSION}

# OpenMPI 4.1.5
OMPI_VERSION="4.1.5"
OMPI_DOWNLOAD_URL=https://download.open-mpi.org/release/open-mpi/v4.1/openmpi-${OMPI_VERSION}.tar.gz
$COMMON_DIR/download_and_verify.sh $OMPI_DOWNLOAD_URL "c018b127619d2a2a30c1931f316fc8a245926d0f5b4ebed4711f9695e7f70925"
tar -xvf openmpi-${OMPI_VERSION}.tar.gz
cd openmpi-${OMPI_VERSION}
./configure --prefix=${INSTALL_PREFIX}/openmpi-${OMPI_VERSION} --with-ucx=${UCX_PATH} --with-hcoll=${HCOLL_PATH} --enable-mpirun-prefix-by-default --with-platform=contrib/platform/mellanox/optimized && make -j$(nproc) && make install
cd ..
$COMMON_DIR/write_component_version.sh "OMPI" ${OMPI_VERSION}

# Intel MPI 2021 (Update 9)
IMPI_2021_VERSION="2021.9.0"
IMPI_2021_DOWNLOAD_URL=https://registrationcenter-download.intel.com/akdlm/IRC_NAS/718d6f8f-2546-4b36-b97b-bc58d5482ebf/l_mpi_oneapi_p_${IMPI_2021_VERSION}.43482_offline.sh
$COMMON_DIR/download_and_verify.sh $IMPI_2021_DOWNLOAD_URL "5c170cdf26901311408809ced28498b630a494428703685203ceef6e62735ef8"
bash l_mpi_oneapi_p_${IMPI_2021_VERSION}.43482_offline.sh -s -a -s --eula accept
mv ${INSTALL_PREFIX}/intel/oneapi/mpi/${IMPI_2021_VERSION}/modulefiles/mpi ${INSTALL_PREFIX}/intel/oneapi/mpi/${IMPI_2021_VERSION}/modulefiles/impi
$COMMON_DIR/write_component_version.sh "IMPI_2021" ${IMPI_2021_VERSION}

# Module Files
MODULE_FILES_DIRECTORY=/usr/share/modules/modulefiles/mpi
mkdir -p ${MODULE_FILES_DIRECTORY}

# HPC-X
cat << EOF >> ${MODULE_FILES_DIRECTORY}/hpcx-${HPCX_VERSION}
#%Module 1.0
#
#  HPCx ${HPCX_VERSION}
#
conflict        mpi
module load ${HPCX_PATH}/modulefiles/hpcx
EOF

# MVAPICH2
cat << EOF >> ${MODULE_FILES_DIRECTORY}/mvapich2-${MV2_VERSION}
#%Module 1.0
#
#  MVAPICH2 ${MV2_VERSION}
#
conflict        mpi
prepend-path    PATH            /opt/mvapich2-${MV2_VERSION}/bin
prepend-path    LD_LIBRARY_PATH /opt/mvapich2-${MV2_VERSION}/lib
prepend-path    MANPATH         /opt/mvapich2-${MV2_VERSION}/share/man
setenv          MPI_BIN         /opt/mvapich2-${MV2_VERSION}/bin
setenv          MPI_INCLUDE     /opt/mvapich2-${MV2_VERSION}/include
setenv          MPI_LIB         /opt/mvapich2-${MV2_VERSION}/lib
setenv          MPI_MAN         /opt/mvapich2-${MV2_VERSION}/share/man
setenv          MPI_HOME        /opt/mvapich2-${MV2_VERSION}
EOF

# OpenMPI
cat << EOF >> ${MODULE_FILES_DIRECTORY}/openmpi-${OMPI_VERSION}
#%Module 1.0
#
#  OpenMPI ${OMPI_VERSION}
#
conflict        mpi
prepend-path    PATH            /opt/openmpi-${OMPI_VERSION}/bin
prepend-path    LD_LIBRARY_PATH /opt/openmpi-${OMPI_VERSION}/lib
prepend-path    MANPATH         /opt/openmpi-${OMPI_VERSION}/share/man
setenv          MPI_BIN         /opt/openmpi-${OMPI_VERSION}/bin
setenv          MPI_INCLUDE     /opt/openmpi-${OMPI_VERSION}/include
setenv          MPI_LIB         /opt/openmpi-${OMPI_VERSION}/lib
setenv          MPI_MAN         /opt/openmpi-${OMPI_VERSION}/share/man
setenv          MPI_HOME        /opt/openmpi-${OMPI_VERSION}
EOF

# Intel 2021
cat << EOF >> ${MODULE_FILES_DIRECTORY}/impi_${IMPI_2021_VERSION}
#%Module 1.0
#
#  Intel MPI ${IMPI_2021_VERSION}
#
conflict        mpi
module load /opt/intel/oneapi/mpi/${IMPI_2021_VERSION}/modulefiles/impi
setenv          MPI_BIN         /opt/intel/oneapi/mpi/${IMPI_2021_VERSION}/bin
setenv          MPI_INCLUDE     /opt/intel/oneapi/mpi/${IMPI_2021_VERSION}/include
setenv          MPI_LIB         /opt/intel/oneapi/mpi/${IMPI_2021_VERSION}/lib
setenv          MPI_MAN         /opt/intel/oneapi/mpi/${IMPI_2021_VERSION}/man
setenv          MPI_HOME        /opt/intel/oneapi/mpi/${IMPI_2021_VERSION}
EOF

# Softlinks
ln -s ${MODULE_FILES_DIRECTORY}/hpcx-${HPCX_VERSION} ${MODULE_FILES_DIRECTORY}/hpcx
ln -s ${MODULE_FILES_DIRECTORY}/mvapich2-${MV2_VERSION} ${MODULE_FILES_DIRECTORY}/mvapich2
ln -s ${MODULE_FILES_DIRECTORY}/openmpi-${OMPI_VERSION} ${MODULE_FILES_DIRECTORY}/openmpi
ln -s ${MODULE_FILES_DIRECTORY}/impi_${IMPI_2021_VERSION} ${MODULE_FILES_DIRECTORY}/impi-2021


#!/bin/bash
set -ex

# Install NCCL
apt install -y build-essential devscripts debhelper fakeroot

case ${DISTRIBUTION} in
    "ubuntu18.04") NCCL_VERSION="2.18.3-1";
        CUDA_VERSION="12.1";;
    "ubuntu20.04") NCCL_VERSION="2.18.3-1";
        CUDA_VERSION="12.1";;
    "ubuntu22.04") NCCL_VERSION="2.18.3-1";
        CUDA_VERSION="12.1";;
    *) ;;
esac

TARBALL="v${NCCL_VERSION}.tar.gz"
NCCL_DOWNLOAD_URL=https://github.com/NVIDIA/nccl/archive/refs/tags/${TARBALL}
pushd /tmp
wget ${NCCL_DOWNLOAD_URL}
tar -xvf ${TARBALL}

pushd nccl-${NCCL_VERSION}
make -j src.build
make pkg.debian.build
pushd build/pkg/deb/
dpkg -i libnccl2_${NCCL_VERSION}+cuda${CUDA_VERSION}_amd64.deb
sudo apt-mark hold libnccl2
dpkg -i libnccl-dev_${NCCL_VERSION}+cuda${CUDA_VERSION}_amd64.deb
sudo apt-mark hold libnccl-dev
popd
popd

# Install the nccl rdma sharp plugin
mkdir -p /usr/local/nccl-rdma-sharp-plugins
apt install -y zlib1g-dev
git clone https://github.com/Mellanox/nccl-rdma-sharp-plugins.git
pushd nccl-rdma-sharp-plugins
./autogen.sh
./configure --prefix=/usr/local/nccl-rdma-sharp-plugins --with-cuda=/usr/local/cuda
make
make install
popd
popd

# Build the nccl tests
source /etc/profile.d/modules.sh
module load mpi/hpcx
git clone https://github.com/NVIDIA/nccl-tests.git
pushd nccl-tests
make MPI=1 MPI_HOME=${HPCX_MPI_DIR} CUDA_HOME=/usr/local/cuda
popd
mv nccl-tests /opt/.
module unload mpi/hpcx

$COMMON_DIR/write_component_version.sh "NCCL" ${NCCL_VERSION}

# Remove installation files
rm -rf /tmp/${TARBALL}
rm -rf /tmp/nccl-${NCCL_VERSION}


#!/bin/bash
set -ex

# Parameters
VERSION=$1

case ${VERSION} in
    1804) NVIDIA_VERSION="525.105.17";
        CUDA_VERSION="12-1";
        CUDA_SAMPLES_VERSION="12.1";
        CHECKSUM="c635a21a282c9b53485f19ebb64a0f4b536a968b94d4d97629e0bc547a58142a";;
    2004) NVIDIA_VERSION="525.105.17";
        CUDA_VERSION="12-1";
        CUDA_SAMPLES_VERSION="12.1";
        CHECKSUM="c635a21a282c9b53485f19ebb64a0f4b536a968b94d4d97629e0bc547a58142a";;
    2204) NVIDIA_VERSION="525.105.17";
        CUDA_VERSION="12-1";
        CUDA_SAMPLES_VERSION="12.1";
        CHECKSUM="c635a21a282c9b53485f19ebb64a0f4b536a968b94d4d97629e0bc547a58142a";;
    *) ;;
esac

# Reference - https://docs.nvidia.com/cuda/cuda-installation-guide-linux/index.html#ubuntu-installation
# Install Cuda
wget https://developer.download.nvidia.com/compute/cuda/repos/ubuntu${VERSION}/x86_64/cuda-keyring_1.0-1_all.deb
dpkg -i ./cuda-keyring_1.0-1_all.deb

apt-get update
apt install -y cuda-toolkit-${CUDA_VERSION}
echo 'export PATH=$PATH:/usr/local/cuda/bin' | tee -a /etc/bash.bashrc
echo 'export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/usr/local/cuda/lib64' | tee -a /etc/bash.bashrc
$COMMON_DIR/write_component_version.sh "CUDA" ${CUDA_VERSION}

# Download CUDA samples
TARBALL="v${CUDA_SAMPLES_VERSION}.tar.gz"
CUDA_SAMPLES_DOWNLOAD_URL=https://github.com/NVIDIA/cuda-samples/archive/refs/tags/${TARBALL}
wget ${CUDA_SAMPLES_DOWNLOAD_URL}
tar -xvf ${TARBALL}
pushd ./cuda-samples-${CUDA_SAMPLES_VERSION}
make
mv -vT ./Samples /usr/local/cuda-${CUDA_SAMPLES_VERSION}/samples
popd

# Nvidia driver
NVIDIA_DRIVER_URL=https://us.download.nvidia.com/tesla/${NVIDIA_VERSION}/NVIDIA-Linux-x86_64-${NVIDIA_VERSION}.run
$COMMON_DIR/download_and_verify.sh $NVIDIA_DRIVER_URL ${CHECKSUM}
bash NVIDIA-Linux-x86_64-${NVIDIA_VERSION}.run --silent --dkms
$COMMON_DIR/write_component_version.sh "NVIDIA" ${NVIDIA_VERSION}


#!/bin/bash
set -ex

# Parameter
# Ubuntu Version
VERSION=$1

# Install nvidia fabric manager
case ${VERSION} in
    1804) NVIDIA_FABRIC_MANAGER_VERSION="525_525.105.17-1";
        CHECKSUM="b487db5923194ba9f4d7c34891f4f8513a3f633a22a0c9f51fba3ef971681977";
        VERSION_PREFIX="525";;
    2004) NVIDIA_FABRIC_MANAGER_VERSION="525_525.105.17-1";
        CHECKSUM="b487db5923194ba9f4d7c34891f4f8513a3f633a22a0c9f51fba3ef971681977";
        VERSION_PREFIX="525";;
    2204) NVIDIA_FABRIC_MANAGER_VERSION="525_525.105.17-1";
        CHECKSUM="b487db5923194ba9f4d7c34891f4f8513a3f633a22a0c9f51fba3ef971681977";
        VERSION_PREFIX="525";;
    *) ;;
esac

NVIDIA_FABRIC_MNGR_URL=http://developer.download.nvidia.com/compute/cuda/repos/ubuntu${VERSION}/x86_64/nvidia-fabricmanager-${NVIDIA_FABRIC_MANAGER_VERSION}_amd64.deb
$COMMON_DIR/download_and_verify.sh $NVIDIA_FABRIC_MNGR_URL ${CHECKSUM}
apt install -y ./nvidia-fabricmanager-${NVIDIA_FABRIC_MANAGER_VERSION}_amd64.deb
apt-mark hold nvidia-fabricmanager-${VERSION_PREFIX}
$COMMON_DIR/write_component_version.sh "NVIDIA_FABRIC_MANAGER" ${NVIDIA_FABRIC_MANAGER_VERSION}


#!/bin/bash
set -ex

# Install NV Peer Memory (GPU Direct RDMA)
sudo apt install -y dkms libnuma-dev
NV_PEER_MEMORY_VERSION="1.3-0"
NV_PEER_MEMORY_VERSION_PREFIX=$(echo ${NV_PEER_MEMORY_VERSION} | awk -F- '{print $1}')
TARBALL="${NV_PEER_MEMORY_VERSION}.tar.gz"
NV_PEER_MEM_DOWNLOAD_URL="https://github.com/Mellanox/nv_peer_memory/archive/refs/tags/${TARBALL}"
wget ${NV_PEER_MEM_DOWNLOAD_URL}
tar -xvf $TARBALL

pushd nv_peer_memory-${NV_PEER_MEMORY_VERSION}
./build_module.sh
popd

pushd /tmp
tar xzf /tmp/nvidia-peer-memory_${NV_PEER_MEMORY_VERSION_PREFIX}.orig.tar.gz
pushd nvidia-peer-memory-${NV_PEER_MEMORY_VERSION_PREFIX}/
# Fix for issue - https://github.com/Mellanox/nv_peer_memory/issues/106
sed -i s/1.2-0/${NV_PEER_MEMORY_VERSION}/g ./debian/changelog
dpkg-buildpackage -us -uc
dpkg -i ../nvidia-peer-memory_${NV_PEER_MEMORY_VERSION}_all.deb
apt-mark hold nvidia-peer-memory
dpkg -i ../nvidia-peer-memory-dkms_${NV_PEER_MEMORY_VERSION}_all.deb
apt-mark hold nvidia-peer-memory-dkms
popd
popd

# load the nvidia-peermem coming as a part of NVIDIA GPU driver
# Reference - https://download.nvidia.com/XFree86/Linux-x86_64/510.85.02/README/nvidia-peermem.html
# Stop nv_peer_mem service
service nv_peer_mem stop
# unload nv_peer_mem
rmmod nv_peer_mem
# load nvidia-peermem
modprobe nvidia-peermem
# verify if loaded
lsmod | grep nvidia_peermem

$COMMON_DIR/write_component_version.sh "NV_PEER_MEMORY" ${NV_PEER_MEMORY_VERSION}


#!/bin/bash
set -ex

apt-get update
apt-get -y install build-essential
apt-get -y install numactl \
                   rpm \
                   libnuma-dev \
                   libmpc-dev \
                   libmpfr-dev \
                   libxml2-dev \
                   m4 \
                   byacc \
                   python-setuptools \
                   tcl \
                   environment-modules \
                   tk \
                   texinfo \
                   libudev-dev \
                   binutils \
                   binutils-dev \
                   selinux-policy-dev \
                   flex \
                   libnl-3-dev \
                   libnl-route-3-dev \
                   libnl-3-200 \
                   libnl-genl-3-dev \
                   libnl-genl-3-200 \
                   bison \
                   libnl-route-3-200 \
                   gfortran \
                   cmake \
                   libnl-3-dev \
                   libnl-route-3-dev \
                   net-tools \
                   libsecret-1-0 \
                   python3-pip \
                   dkms \
                   jq \
                   curl \
                   libyaml-dev \
                   libreadline-dev \
                   libkeyutils1 \
                   libkeyutils-dev \
                   libmount-dev \
                   nfs-common \
                   pssh

if [[ $DISTRIBUTION != "ubuntu22.04" ]]; then apt-get install -y python-dev; fi

# Install azcopy tool
# To copy blobs or files to or from a storage account.
# Parameters - Version, Release Tag
VERSION=$1
RELEASE_TAG=$2
TARBALL="azcopy_linux_amd64_${VERSION}.tar.gz"
AZCOPY_DOWNLOAD_URL="https://azcopyvnext.azureedge.net/${RELEASE_TAG}/${TARBALL}"
AZCOPY_FOLDER=$(basename ${AZCOPY_DOWNLOAD_URL} .tgz)
wget ${AZCOPY_DOWNLOAD_URL}
tar -xvf ${TARBALL}

# copy the azcopy to the bin path
pushd azcopy_linux_amd64_${VERSION}
cp azcopy /usr/bin/
popd

# Allow execute permissions
chmod +x /usr/bin/azcopy

# remove tarball from azcopy
rm -rf *.tar.gz

# copy kvp client file
$COMMON_DIR/copy_kvp_client.sh


#!/bin/bash
set -ex

source /etc/lsb-release

echo "deb [arch=amd64] https://packages.microsoft.com/repos/amlfs-${DISTRIB_CODENAME}/ ${DISTRIB_CODENAME} main" | sudo tee /etc/apt/sources.list.d/amlfs.list

# Enable these lines if the MS PMC repo was not already setup.
#curl https://packages.microsoft.com/keys/microsoft.asc | gpg --dearmor > microsoft.gpg
#cp ./microsoft.gpg /etc/apt/trusted.gpg.d/


#!/bin/bash
set -ex

# set properties
source ./set_properties.sh

# install utils
./install_utils.sh

# install Lustre client
$UBUNTU_COMMON_DIR/install_lustre_client.sh

# install mellanox ofed
./install_mellanoxofed.sh

# install mpi libraries
./install_mpis.sh

# install nvidia gpu driver
./install_nvidiagpudriver.sh

# Install NCCL
$UBUNTU_COMMON_DIR/install_nccl.sh

# Install NVIDIA docker container
$UBUNTU_COMMON_DIR/install_docker.sh

# cleanup downloaded tarballs - clear some space
rm -rf *.tgz *.bz2 *.tbz *.tar.gz *.run *.deb *_offline.sh
rm -rf /tmp/MLNX_OFED_LINUX* /tmp/*conf*
rm -rf /var/intel/ /var/cache/*
rm -Rf -- */

# Install DCGM
$UBUNTU_COMMON_DIR/install_dcgm.sh

# install Intel libraries
$UBUNTU_COMMON_DIR/install_intel_libs.sh

# install diagnostic script
$COMMON_DIR/install_hpcdiag.sh

# install persistent rdma naming
$COMMON_DIR/install_azure_persistent_rdma_naming.sh

# optimizations
$UBUNTU_COMMON_DIR/hpc-tuning.sh

# copy test file
$COMMON_DIR/copy_test_file.sh

# install monitor tools
$UBUNTU_COMMON_DIR/install_monitoring_tools.sh

# diable auto kernel updates
$UBUNTU_COMMON_DIR/disable_auto_upgrade.sh

# SKU Customization
$COMMON_DIR/setup_sku_customizations.sh

# clear history
# Uncomment the line below if you are running this on a VM
# $COMMON_DIR/clear_history.sh


#!/bin/bash
set -ex

VERSION="23.04-1.1.3.0"
TARBALL="MLNX_OFED_LINUX-$VERSION-ubuntu18.04-x86_64.tgz"
MLNX_OFED_DOWNLOAD_URL=https://content.mellanox.com/ofed/MLNX_OFED-${VERSION}/$TARBALL
MOFED_FOLDER=$(basename ${MLNX_OFED_DOWNLOAD_URL} .tgz)

$COMMON_DIR/download_and_verify.sh $MLNX_OFED_DOWNLOAD_URL "634960b96698f845ccc8974987c71dfeced4553ddc5fb0aa1a79071307cbdd45"
tar zxvf ${TARBALL}

./${MOFED_FOLDER}/mlnxofedinstall --add-kernel-support --skip-unsupported-devices-check --without-fw-update
$COMMON_DIR/write_component_version.sh "MOFED" $VERSION

# Restarting openibd
/etc/init.d/openibd restart

## Fix for systemd-modules-load service failing on boot
rm -rf /lib/modules/$(uname -r)/kernel/drivers/infiniband/ulp/iser/ib_iser.ko
depmod


#!/bin/bash
set -ex

# Install common MPIs for Ubuntu
$UBUNTU_COMMON_DIR/install_mpis.sh "c8868382a7b93286cd70ac46c63489f176bbfe0d76b3e5b2b91ecc0f93272414"


#!/bin/bash
set -ex

$UBUNTU_COMMON_DIR/install_nvidiagpudriver.sh 1804

# Install gdrcopy
sudo apt install -y build-essential devscripts debhelper check libsubunit-dev fakeroot pkg-config dkms
GDRCOPY_VERSION="2.3"
TARBALL="v${GDRCOPY_VERSION}.tar.gz"
GDRCOPY_DOWNLOAD_URL=https://github.com/NVIDIA/gdrcopy/archive/refs/tags/${TARBALL}
wget $GDRCOPY_DOWNLOAD_URL
tar -xvf $TARBALL

pushd gdrcopy-${GDRCOPY_VERSION}/packages/
CUDA=/usr/local/cuda ./build-deb-packages.sh
sudo dpkg -i gdrdrv-dkms_${GDRCOPY_VERSION}-1_amd64.Ubuntu18_04.deb
sudo apt-mark hold gdrdrv-dkms
sudo dpkg -i libgdrapi_${GDRCOPY_VERSION}-1_amd64.Ubuntu18_04.deb
sudo apt-mark hold libgdrapi
sudo dpkg -i gdrcopy-tests_${GDRCOPY_VERSION}-1_amd64.Ubuntu18_04.deb
sudo apt-mark hold gdrcopy-tests
sudo dpkg -i gdrcopy_${GDRCOPY_VERSION}-1_amd64.Ubuntu18_04.deb
sudo apt-mark hold gdrcopy
popd

$COMMON_DIR/write_component_version.sh "GDRCOPY" ${GDRCOPY_VERSION}

# Install nvidia fabric manager (required for ND96asr_v4)
$UBUNTU_COMMON_DIR/install_nvidia_fabric_manager.sh 1804


#!/bin/bash
set -ex

# Setup microsoft packages repository for moby
# Download the repository configuration package
curl https://packages.microsoft.com/config/ubuntu/18.04/prod.list > ./microsoft-prod.list
# Copy the generated list to the sources.list.d directory
cp ./microsoft-prod.list /etc/apt/sources.list.d/
# Install the Microsoft GPG public key
curl https://packages.microsoft.com/keys/microsoft.asc | gpg --dearmor > microsoft.gpg
cp ./microsoft.gpg /etc/apt/trusted.gpg.d/

#install apt pckages
AZCOPY_VERSION="10.16.2"
AZCOPY_RELEASE_TAG="release20221108"
$UBUNTU_COMMON_DIR/install_utils.sh ${AZCOPY_VERSION} ${AZCOPY_RELEASE_TAG}

apt-get update
apt-get install -y python3.8
update-alternatives --install /usr/bin/python3 python3 /usr/bin/python3.8 1
apt-get -y install --reinstall python3-apt

apt-get -y install python3-pip
DISTPACK=/usr/lib/python3/dist-packages
cp $DISTPACK/apt_pkg.cpython-36m-x86_64-linux-gnu.so $DISTPACK/apt_pkg.so
apt-get install -y libcairo2-dev
apt-get install -y python3-dev
apt-get install -y libpython3.8-dev
apt-get install -y libgirepository1.0-dev
python3.8 -m pip install --ignore-installed PyGObject
apt-get install -y software-properties-common

# For networkd-dispatcher + unattended-upgrades services to work correctly. Specific to ubunut 18.04
ln -sf  /usr/lib/python3/dist-packages/_dbus_glib_bindings.cpython-36m-x86_64-linux-gnu.so /usr/lib/python3/dist-packages/_dbus_glib_bindings.so
ln -sf  /usr/lib/python3/dist-packages/_dbus_bindings.cpython-36m-x86_64-linux-gnu.so /usr/lib/python3/dist-packages/_dbus_bindings.so
apt-get -y install libglib2.0-dev libdbus-1-3 libdbus-1-dev

sudo python3 -m  pip install meson ninja
sudo python3 -m pip install pgi dbus-python


#!/bin/bash

export TOP_DIR=../../..
export COMMON_DIR=../../../common
export UBUNTU_COMMON_DIR=../../common
export TEST_DIR=../../../tests
export DISTRIBUTION=$(. /etc/os-release;echo $ID$VERSION_ID)


#!/bin/bash
set -ex

# set properties
source ./set_properties.sh

# install utils
./install_utils.sh

# install Lustre client
$UBUNTU_COMMON_DIR/install_lustre_client.sh

# install mellanox ofed
./install_mellanoxofed.sh

# install mpi libraries
./install_mpis.sh

# install nvidia gpu driver
./install_nvidiagpudriver.sh

# Install NCCL
./install_nccl.sh

# cleanup downloaded tarballs
rm -rf *.tgz *.bz2 *.tbz *.tar.gz *.run *.deb *_offline.sh
rm -Rf -- */

# Install DCGM
$UBUNTU_COMMON_DIR/install_dcgm.sh

# install Intel libraries
$UBUNTU_COMMON_DIR/install_intel_libs.sh

# install diagnostic script
$COMMON_DIR/install_hpcdiag.sh

# optimizations
$UBUNTU_COMMON_DIR/hpc-tuning.sh

# SKU Customization
$COMMON_DIR/setup_sku_customizations.sh

# copy test file
$COMMON_DIR/copy_test_file.sh

# diable auto kernel updates
$UBUNTU_COMMON_DIR/disable_auto_upgrade.sh

# clear history
# Uncomment the line below if you are running this on a VM
# $COMMON_DIR/clear_history.sh


#!/bin/bash
set -ex

VERSION="4.9-6.0.6.0"
TARBALL="MLNX_OFED_LINUX-$VERSION-ubuntu18.04-x86_64.tgz"
MLNX_OFED_DOWNLOAD_URL=https://content.mellanox.com/ofed/MLNX_OFED-${VERSION}/$TARBALL
MOFED_FOLDER=$(basename ${MLNX_OFED_DOWNLOAD_URL} .tgz)

$COMMON_DIR/download_and_verify.sh $MLNX_OFED_DOWNLOAD_URL "ca56ddf53a22192d2c91d1c410db686a8f8faf339020cfe44ab0c664e63c8b99"
tar zxvf ${TARBALL}

./${MOFED_FOLDER}/mlnxofedinstall --add-kernel-support --without-fw-update
$COMMON_DIR/write_component_version.sh "MOFED" $VERSION

# Restarting openibd
/etc/init.d/openibd restart