Siesta 4.1.5 MPI (HPE)
Webpagehttps://gitlab.com/siesta-project/siesta

Version4.1.5 (+ELPA 2021.05.002)

Build EnvironmentIntel oneAPI Compiler Classic 2022.2.1
Intel MKL 2022.2.1
HPC-X 2.13.1 (Open MPI 4.1.5)

Files Required
siesta-v4.1.5.tar.gz
arch.make.SUFFIXES:.SUFFIXES: .f .F .o .c .a .f90 .F90
SIESTA_ARCH = rccs-intel2022-mkl
CC = mpicc
FPP = $(FC) -E -P -x c
FC = mpif90
FC_SERIAL = ifort
FFLAGS = -O2 -fPIC -march=core-avx2 -fp-model source
AR = ar
ARFLAGS_EXTRA =RANLIB = ranlib
SYS = nag
SP_KIND = 4
DP_KIND = 8
KINDS = $(SP_KIND) $(DP_KIND)
DEFS_PREFIX =LDFLAGS =FCFLAGS_fixed_f = -fixed
FCFLAGS_free_f90 = -free
FPPFLAGS_fixed_F = -fixed
FPPFLAGS_free_F90 = -free
BLAS_LIBS = -mkl=sequential
LAPACK_LIBS = -mkl=sequential
SCALAPACK_LIBS = -L$(MKLROOT)/lib/intel64 -lmkl_scalapack_lp64 -lmkl_blacs_openmpi_lp64 -lmkl_intel_lp64 -lmkl_sequential -lmkl_core -lpthread -lm-ldl
COMP_LIBS =NETCDF_ROOT = /apl/siesta/4.1.5-mpi/exts
NETCDF_INCFLAGS = -I$(NETCDF_ROOT)/include
NETCDF_LIBS = -L$(NETCDF_ROOT)/lib -lnetcdff -lnetcdf
MPI_INTERFACE = libmpi_f90.a
MPI_INCLUDE = .

FPPFLAGS = $(DEFS_PREFIX)-DFC_HAVE_ABORT -DSIESTA__ELPA -DMPI -DCDF -DFC_HAVE_ABORT -DFC_HAVE_FLUSH
LIBS = $(NETCDF_LIBS) $(SCALAPACK_LIBS) $(LAPACK_LIBS) $(MPI_LIBS) $(COMP_LIBS)
FFLAGS_DEBUG = -g -O1 -fp-model source# ELPA
ELPA_ROOT = /apl/siesta/4.1.5-mpi/exts
ELPA_INCFLAGS = -DSIESTA__ELPA -I$(ELPA_ROOT)/include/elpa-2021.05.002/modules
ELPA_LIBS = -L$(ELPA_ROOT)/lib -lelpa
LIBS += $(ELPA_LIBS)
FPPFLAGS += $(ELPA_INCFLAGS)
atom.o: atom.F$(FC) -c $(FFLAGS_DEBUG) $(INCFLAGS) $(FPPFLAGS) $(FPPFLAGS_fixed_F) $<state_analysis.o: state_analysis.F$(FC) -c $(FFLAGS_DEBUG) $(INCFLAGS) $(FPPFLAGS) $(FPPFLAGS_fixed_F) $<.c.o:$(CC) -c $(CFLAGS) $(INCFLAGS) $(CPPFLAGS) $<.F.o:$(FC) -c $(FFLAGS) $(INCFLAGS) $(FPPFLAGS) $(FPPFLAGS_fixed_F) $<.F90.o:$(FC) -c $(FFLAGS) $(INCFLAGS) $(FPPFLAGS) $(FPPFLAGS_free_F90) $<.f.o:$(FC) -c $(FFLAGS) $(INCFLAGS) $(FCFLAGS_fixed_f) $<.f90.o:$(FC) -c $(FFLAGS) $(INCFLAGS) $(FCFLAGS_free_f90) $<elpa-2021.05.002.tar.gz
netcdf-c-4.8.1.tar.gz
netcdf-fortran-4.5.3.tar.gz

Build Procedure
ELPA 2021.05.002#!/bin/sh
ELPA_VERSION=2021.05.002
INSTDIR=/apl/siesta/4.1.5-mpi/exts
WORKDIR=/gwork/users/${USER}BASEDIR=/home/users/${USER}/Software/ELPA/${ELPA_VERSION}TARBALL=${BASEDIR}/elpa-${ELPA_VERSION}.tar.gz
PARALLEL=12#--------------------------------------------------------------------------umask 0022
ulimit -s unlimited
module purge. ~/intel/oneapi/compiler/2022.2.1/env/vars.sh
module load compiler-rt/2022.2.1
module load mkl/2022.2.1
module load openmpi/4.1.5-hpcx/intel2022.2.1
export LANG=C
export LC_ALL=Cexport FC=mpif90
export CC=mpicc
export CXX=mpicxx

export FCFLAGS=-I${MKLROOT}/include/intel64/lp64/# mkl_link_tool -libs -c intel_f -p no --cluster_library=scalapack
export LDFLAGS="-L${MKLROOT}/lib/intel64 -lmkl_scalapack_lp64 -lmkl_blacs_openmpi_lp64 -lmkl_intel_lp64 -lmkl_sequential -lmkl_core -lpthread -lm-ldl"
cd ${WORKDIR}
if [ -d elpa-${ELPA_VERSION} ]; then
mv elpa-${ELPA_VERSION} elpa-erase
rm -rf elpa-erase &fitar zxf ${TARBALL}
cd elpa-${ELPA_VERSION}./configure --prefix=${INSTDIR} --disable-avx512
make -j ${PARALLEL}
make checkmake install

Siesta#!/bin/sh
SIESTA_VERSION=4.1.5
INSTDIR=/apl/siesta/4.1.5-mpi
WORKDIR=/gwork/users/${USER}BASEDIR=/home/users/${USER}/Software/Siesta/${SIESTA_VERSION}TARBALL=${BASEDIR}/siesta-v${SIESTA_VERSION}.tar.gz
ARCHMAKE=${BASEDIR}-hpe/arch.make
NETCDF_C_VERSION=4.8.1
NETCDF_F_VERSION=4.5.3
BASEDIR_NETCDF=/home/users/${USER}/Software/NETCDFTARBALL_NETCDF_C=${BASEDIR_NETCDF}/c${NETCDF_C_VERSION}/netcdf-c-${NETCDF_C_VERSION}.tar.gz
TARBALL_NETCDF_F=${BASEDIR_NETCDF}/f${NETCDF_F_VERSION}/netcdf-fortran-${NETCDF_F_VERSION}.tar.gz
PARALLEL=12 # NOTE: parallel make cannot be used for siesta#--------------------------------------------------------------------------umask 0022
ulimit -s unlimited
module -s purge. ~/intel/oneapi/compiler/2022.2.1/env/vars.sh
module -s load compiler-rt/2022.2.1
module -s load mkl/2022.2.1
module -s load openmpi/4.1.5-hpcx/intel2022.2.1
export LANG=C
export LC_ALL=C
export FC=ifort
export CC=icc
cd ${WORKDIR}
if [ -d netcdf-c-${NETCDF_C_VERSION} ]; then
mv netcdf-c-${NETCDF_C_VERSION} netcdf-c-erase
rm -rf netcdf-c-erase &fi
tar zxf ${TARBALL_NETCDF_C}
cd netcdf-c-${NETCDF_C_VERSION}./configure --prefix=${INSTDIR}/exts
make -j${PARALLEL}# i20u2: failed on tst_charvlenbug.cmake -j${PARALLEL} check # can pass correctlymake installexport PATH="${PATH}:${INSTDIR}/exts/bin"
export CPATH="${CPATH}:${INSTDIR}/exts/include"
export LD_LIBRARY_PATH="${LD_LIBRARY_PATH}:${INSTDIR}/exts/lib"
export LIBRARY_PATH="${LIBRARY_PATH}:${INSTDIR}/exts/lib"
cd ${WORKDIR}
if [ -d netcdf-fortran-${NETCDF_F_VERSION} ]; then
mv netcdf-fortran-${NETCDF_F_VERSION} netcdf-fortran-erase
rm -rf netcdf-fortran-erase &fitar zxf ${TARBALL_NETCDF_F}
cd netcdf-fortran-${NETCDF_F_VERSION}./configure --prefix=${INSTDIR}/exts
make -j${PARALLEL}
make -j${PARALLEL} checkmake install
cd ${INSTDIR}if [ -d siesta-v${SIESTA_VERSION} ]; thenmv siesta-v${SIESTA_VERSION} siesta-erase
rm -rf siesta-erase
fitar zxf ${TARBALL}
mv siesta-v${SIESTA_VERSION}/* .
rmdir siesta-v${SIESTA_VERSION}# hidoiyo...
echo >> Tests/OMM_h2o/OMM_h2o.fdf
echo >> Tests/OMM_si/OMM_si.fdf
mkdir bin # install dir
cd Obj../Src/obj_setup.sh
cp ${ARCHMAKE} ./arch.make# build transiesta
cd ${INSTDIR}/Obj && make transiesta# build siesta
make clean-transiesta && make# utils
cd ${INSTDIR}/Utilecho "m_cite.o: version.o" >> Gen-basis/Makefile
sh build_all.sh# test siesta & transiesta
cd ${INSTDIR}/Obj/Tests
make MPI="mpirun -np 2" SIESTA="${INSTDIR}/Obj/siesta" check >& make_check.log
make MPI="mpirun -np 2" TS="${INSTDIR}/Obj/transiesta" tests-ts >& make_check_ts.log
cd ../
mv siesta ${INSTDIR}/bin
mv transiesta ${INSTDIR}/bin

Notes
Frozen occasionally (for some time?). The cause is unknown. This happens not only for multi node calculations but
also for single node ones.
Intel MPI version does not work at all, always frozen during the calculation.
In the previous system, this error was not observed, although the same siesta and elpa version were used.
MKL version is more likely to freeze than normal scalapack version? Anyways, non-MKL version still freezes
occasionally.
Switching HPC-X 2.11 or rebuilding with HPC-X 2.11 don't help.