NWChem-7.2.2 with CUDA
ウェブページ
https://nwchemgit.github.io/

バージョン
7.2.2

ビルド環境
GCC 12.1.1 (gcc-toolset-12)
Intel MKL 2023.2.0
Open MPI 4.1.6

ビルドに必要なファイル
nwchem-7.2.2-release.revision-74936fb9-src.2023-11-03.tar.bz2
runtest.md.mpi#!/bin/sh./runtests.mpi.unix procs 48 \na_k/nak \na_k/nak_md \crown/crown_md \ethanol/ethanol_md \ethanol/ethanol_ti \had/had_em \had/had_md \prep/a3n \prep/aal \prep/fsc \water/water_md

ビルド手順#!/bin/shVERSION=7.2.2
INSTALL_PREFIX=/apl/nwchem/7.2.2-CUDA
BASEDIR=/home/users/${USER}/Software/NWChem/${VERSION}TARBALL=${BASEDIR}/nwchem-7.2.2-release.revision-74936fb9-src.2023-11-03.tar.bz2
WORKDIR=/gwork/users/${USER}#--------------------------------------------------------------------umask 0022
export LANG=C
ulimit -s unlimited
module -s purge
module -s load gcc-toolset/12
module -s load mkl/2023.2.0
module -s load openmpi/4.1.6/gcc12
module -s load cuda/12.1u1
cd ${WORKDIR}
if [ -d nwchem-${VERSION} ]; then
mv nwchem-${VERSION} nwchem-erase
rm -rf nwchem-erase &fi

tar jxf ${TARBALL}
export NWCHEM_TOP=${WORKDIR}/nwchem-${VERSION}
export NWCHEM_MODULES="all python"
export NWCHEM_TARGET=LINUX64
export ARMCI_NETWORK=MPI-PR
export USE_OPENMP=yexport USE_MPI=yexport USE_MPIF=yexport USE_MPIF4=yexport TCE_CUDA=yexport CUDA_FLAGS="-arch sm_80"
export CUDA_LIBS="-lcudart"
export USE_NOFSCHECK=TRUE
export USE_NOIO=TRUE
export MRCC_METHODS=TRUE
export CCSDTQ=TRUE
export LIB_DEFINES=-DDFLT_TOT_MEM=180000000 # 1.8 GB/processexport PYTHONVERSION=3.6
export BLAS_SIZE=8
export BLASOPT="-L${MKLROOT}/lib/intel64 -lmkl_intel_ilp64 -lmkl_gnu_thread -lmkl_core -liomp5 -lpthread -lm -ldl"
export LAPACK_SIZE=8
export LAPACK_LIB="${BLASOPT}"
export USE_SCALAPACK=yexport SCALAPACK_SIZE=8
export SCALAPACK="-L${MKLROOT}/lib/intel64 -lmkl_scalapack_ilp64 -lmkl_blacs_openmpi_ilp64 -lmkl_intel_ilp64 -lmkl_gnu_thread -lmkl_core -liomp5-lpthread -lm -ldl"
cd ${NWCHEM_TOP}/src# cheatmkdir ../binln -s /usr/libexec/platform-python3.6-config ../bin/python3.6-config
export PATH="${PATH}:${NWCHEM_TOP}/bin:${NWCHEM_TOP}/QA"
make nwchem_config
make# involve also version info
cd ${NWCHEM_TOP}/src/util
make version
make
cd ${NWCHEM_TOP}/src
make link
cd ${NWCHEM_TOP}# installation; assume INSTALL_PREFIX directory already exists
cp -fr LICENSE.TXT README.md release.notes.* examples ${INSTALL_PREFIX}
mkdir ${INSTALL_PREFIX}/bin
mkdir ${INSTALL_PREFIX}/data
cp -f ${NWCHEM_TOP}/bin/${NWCHEM_TARGET}/nwchem ${INSTALL_PREFIX}/bin
chmod 755 ${INSTALL_PREFIX}/bin/nwchem
cp -fr ${NWCHEM_TOP}/src/data${INSTALL_PREFIX}

cp -fr ${NWCHEM_TOP}/src/basis/libraries${INSTALL_PREFIX}/data

cp -fr ${NWCHEM_TOP}/src/basis/libraries.bse ${INSTALL_PREFIX}/data
cp -fr ${NWCHEM_TOP}/src/nwpw/libraryps${INSTALL_PREFIX}/data# create default.nwchemrc
cat << EOS > ${INSTALL_PREFIX}/default.nwchemrc
nwchem_basis_library ${INSTALL_PREFIX}/data/libraries/
nwchem_nwpw_library ${INSTALL_PREFIX}/data/libraryps/
ffield amber
amber_1 ${INSTALL_PREFIX}/data/amber_s/amber_2 ${INSTALL_PREFIX}/data/amber_x/#amber_3 ${INSTALL_PREFIX}/data/amber_q/#amber_4 ${INSTALL_PREFIX}/data/amber_u/
spce${INSTALL_PREFIX}/data/solvents/spce.rst

charmm_s ${INSTALL_PREFIX}/data/charmm_s/
charmm_x ${INSTALL_PREFIX}/data/charmm_x/
EOS# some tests need this...
cp -f ${INSTALL_PREFIX}/default.nwchemrc ~/.nwchemrc

テスト
以下のテストを実行。(GPU を使うものは tce_cuda テストのみ)#!/bin/sh#PBS -l select=1:ncpus=16:mpiprocs=16:ompthreads=1:ngpus=1#PBS -l walltime=24:00:00
VERSION=7.2.2
INSTALL_PREFIX=/apl/nwchem/7.2.2-CUDA
BASEDIR=/home/users/${USER}/Software/NWChem/${VERSION}TARBALL=${BASEDIR}/nwchem-7.2.2-release.revision-74936fb9-src.2023-11-03.tar.bz2
WORKDIR=/gwork/users/${USER}#--------------------------------------------------------------------umask 0022
export LANG=Culimit -s unlimited
module -s purge
module -s load gcc-toolset/12
module -s load mkl/2023.2.0
module -s load openmpi/4.1.6/gcc12
module -s load cuda/12.1u1
export NWCHEM_TOP=${WORKDIR}/nwchem-${VERSION}# run test
export NWCHEM_EXECUTABLE=${INSTALL_PREFIX}/bin/nwchem
cd ${NWCHEM_TOP}/QA./doqmtests.mpi 2 fast >& doqmtests.mpi.fast.log
mv testoutputs testoutputs-serial
mkdir -p ${INSTALL_PREFIX}/testlog
cp -fr doqmtests.mpi.fast.log \testoutputs-serial \${INSTALL_PREFIX}/testlog#!/bin/sh#PBS -l select=1:ncpus=32:mpiprocs=32:ompthreads=1:ngpus=2#PBS -l walltime=24:00:00
VERSION=7.2.2
INSTALL_PREFIX=/apl/nwchem/7.2.2-CUDA

BASEDIR=/home/users/${USER}/Software/NWChem/${VERSION}TARBALL=${BASEDIR}/nwchem-7.2.2-release.revision-74936fb9-src.2023-11-03.tar.bz2
WORKDIR=/gwork/users/${USER}RUNTESTMD=runtest.md.mpi
RUNTESTMD_PATH=${BASEDIR}/${RUNTESTMD}#--------------------------------------------------------------------umask 0022
export LANG=C
ulimit -s unlimited
module -s purge
module -s load gcc-toolset/12
module -s load mkl/2023.2.0
module -s load openmpi/4.1.6/gcc12
module -s load cuda/12.1u1
export NWCHEM_TOP=${WORKDIR}/nwchem-${VERSION}# run test
export NWCHEM_EXECUTABLE=${INSTALL_PREFIX}/bin/nwchem
cd ${NWCHEM_TOP}/QA./doqmtests.mpi 32 >& doqmtests.mpi.log
sed -e "s/48/32/" ${RUNTESTMD_PATH} > ${RUNTESTMD}
sh ${RUNTESTMD} >& runtest.md.mpi.log
mv testoutputs testoutputs-mpi
mkdir -p ${INSTALL_PREFIX}/testlog
cp -fr doqmtests.mpi.log \runtest.md.mpi.log \testoutputs-mpi \${INSTALL_PREFIX}/testlog

テスト結果
テスト結果は /apl/nwchem/7.2.2-CUDA/testlog 以下に格納されています。
失敗したシリアル版テスト
tests/h2o-b3lyp-disp/h2o-b3lyp-disp: メモリ不足(tests/oh2/oh2: 失敗するべきテスト)
tests/dft_siosi3/dft_siosi3: メモリ不足
tests/pspw_md/pspw_md: マイナーな数値エラー(-14.38970 vs -14.38971)
tests/tce_cr_eom_t_ozone/tce_cr_eom_t_ozone: createfile: failed ga_create size/nproc bytes:
tests/qmmm_grad0/qmmm_grad0: メモリ不足
tests/lys_qmmm/lys_qmmm: 同上
tests/ethane_qmmm/ethane_qmmm: 同上
tests/qmmm_freq/qmmm_freq: 同上
tests/h2o-b3lyp-disp/h2o-b3lyp-disp: メモリ不足
tests/o2_ccca/o2_ccca: createfile: failed ga_create size/nproc bytes:失敗した並列版テスト
tests/h2o-b3lyp-disp/h2o-b3lyp-disp: メモリ不足(tests/oh2/oh2: 失敗するべきテスト)
tests/qmmm_grad0/qmmm_grad0: メモリ不足
tests/lys_qmmm/lys_qmmm: 同上
tests/ethane_qmmm/ethane_qmmm: 同上
tests/qmmm_freq/qmmm_freq: 同上
tests/h2o-b3lyp-disp/h2o-b3lyp-disp: 同上
MDテスト
tests/na_k/nak_md: ?

tests/crown/crown_md: Unable to read previous box pair list
tests/ethanol/ethanol_md: ethanol_md.err: No such file or directory
tests/ethanol/ethanol_ti: Unable to read previous box pair list
tests/had/had_em: 0: Dimension mwm too small 0
tests/had/had_md: Unresolved atom types in fragment HEM
tests/prep/a3n: Unresolved atom types in fragment HEM

メモ
CPU 版の記事の内容をご確認ください。
GPU は TCE モジュール内のでのみ利用可能です。
(CPU 版とは別バイナリとして作成)