Repository: NVIDIA/hpc-container-maker

Issue:
NetCDF issues
Hi guys, I'm having troubles installing NetCDF with HPCCM. The issue is that the default version of netcdf-c library is now 4.7.3, and the later releases are not available via ftp anymore, see ftp://ftp.unidata.ucar.edu/pub/netcdf/
Specifying explicit version='4.7.3' works fine.

A different solution would also be to change the FTP web-address to the one directly from github releases (there the version archive is still maintained): https://github.com/Unidata/netcdf-c/releases

Exactly same stuff with versions apply to netcdf-fortran and netcdf-cxx

Cheers!

Discussion:

Patch:
diff --git a/hpccm/building_blocks/netcdf.py b/hpccm/building_blocks/netcdf.py
index c18e907..3b074e9 100644
--- a/hpccm/building_blocks/netcdf.py
+++ b/hpccm/building_blocks/netcdf.py
@@ -122,7 +122,9 @@ class netcdf(bb_base, hpccm.templates.ConfigureMake, hpccm.templates.envvars,
 
         self.configure_opts = kwargs.get('configure_opts', [])
 
-        self.__baseurl = 'ftp://ftp.unidata.ucar.edu/pub/netcdf'
+        self.__c_baseurl = 'https://github.com/Unidata/netcdf-c/archive'
+        self.__cxx_baseurl = 'https://github.com/Unidata/netcdf-cxx4/archive'
+        self.__fortran_baseurl = 'https://github.com/Unidata/netcdf-fortran/archive'
         self.__check = kwargs.get('check', False)
         self.__cxx = kwargs.get('cxx', True)
         self.__fortran = kwargs.get('fortran', True)
@@ -209,13 +211,15 @@ class netcdf(bb_base, hpccm.templates.ConfigureMake, hpccm.templates.envvars,
         if not toolchain.LDFLAGS:
             toolchain.LDFLAGS = '-L{}/lib'.format(self.__hdf5_dir)
 
-        # Version 4.7.0 changed the package name
-        if LooseVersion(self.__version) >= LooseVersion('4.7.0'):
+        # Version 4.3.1 changed the package name
+        if LooseVersion(self.__version) >= LooseVersion('4.3.1'):
             pkgname = 'netcdf-c'
+            tarball = 'v{0}.tar.gz'.format(self.__version)
         else:
             pkgname = 'netcdf'
-        tarball = '{0}-{1}.tar.gz'.format(pkgname, self.__version)
-        url = '{0}/{1}'.format(self.__baseurl, tarball)
+            tarball = '{0}-{1}.tar.gz'.format(pkgname, self.__version)
+
+        url = '{0}/{1}'.format(self.__c_baseurl, tarball)
 
         # Download source from web
         self.__commands.append(self.download_step(url=url,
@@ -257,16 +261,23 @@ class netcdf(bb_base, hpccm.templates.ConfigureMake, hpccm.templates.envvars,
         # without impacting the original.
         toolchain = _copy(self.__toolchain)
 
-        # Need to tell it where to find NetCDF
+        # Need to tell it where to find NetCDF and HDF5
         if not toolchain.CPPFLAGS:
-            toolchain.CPPFLAGS = '-I{}/include'.format(self.prefix)
+            toolchain.CPPFLAGS = '-I{0}/include -I{1}/include'.format(self.prefix, self.__hdf5_dir)
         if not toolchain.LDFLAGS:
             toolchain.LDFLAGS = '-L{}/lib'.format(self.prefix)
         if not toolchain.LD_LIBRARY_PATH:
             toolchain.LD_LIBRARY_PATH = '{}/lib:$LD_LIBRARY_PATH'.format(self.prefix)
 
-        tarball = '{0}-{1}.tar.gz'.format(pkg, version)
-        url = '{0}/{1}'.format(self.__baseurl, tarball)
+        if pkg == 'netcdf-cxx4':
+            baseurl = self.__cxx_baseurl
+        elif pkg == 'netcdf-fortran':
+            baseurl = self.__fortran_baseurl
+        else:
+            raise RuntimeError('unrecognized package name: "{}"'.format(pkg))
+
+        tarball = 'v{0}.tar.gz'.format(version)
+        url = '{0}/{1}'.format(baseurl, tarball)
 
         # Download source from web
         self.__commands.append(self.download_step(url=url,


Test patch:
diff --git a/test/test_netcdf.py b/test/test_netcdf.py
index d944a1d..926d484 100644
--- a/test/test_netcdf.py
+++ b/test/test_netcdf.py
@@ -49,24 +49,24 @@ RUN apt-get update -y && \
         wget \
         zlib1g-dev && \
     rm -rf /var/lib/apt/lists/*
-RUN mkdir -p /var/tmp && wget -q -nc --no-check-certificate -P /var/tmp ftp://ftp.unidata.ucar.edu/pub/netcdf/netcdf-c-4.7.0.tar.gz && \
-    mkdir -p /var/tmp && tar -x -f /var/tmp/netcdf-c-4.7.0.tar.gz -C /var/tmp -z && \
+RUN mkdir -p /var/tmp && wget -q -nc --no-check-certificate -P /var/tmp https://github.com/Unidata/netcdf-c/archive/v4.7.0.tar.gz && \
+    mkdir -p /var/tmp && tar -x -f /var/tmp/v4.7.0.tar.gz -C /var/tmp -z && \
     cd /var/tmp/netcdf-c-4.7.0 &&  CPPFLAGS=-I/usr/local/hdf5/include LDFLAGS=-L/usr/local/hdf5/lib ./configure --prefix=/usr/local/netcdf && \
     make -j$(nproc) && \
     make -j$(nproc) install && \
-    rm -rf /var/tmp/netcdf-c-4.7.0.tar.gz /var/tmp/netcdf-c-4.7.0 && \
-    mkdir -p /var/tmp && wget -q -nc --no-check-certificate -P /var/tmp ftp://ftp.unidata.ucar.edu/pub/netcdf/netcdf-cxx4-4.3.0.tar.gz && \
-    mkdir -p /var/tmp && tar -x -f /var/tmp/netcdf-cxx4-4.3.0.tar.gz -C /var/tmp -z && \
-    cd /var/tmp/netcdf-cxx4-4.3.0 &&  CPPFLAGS=-I/usr/local/netcdf/include LD_LIBRARY_PATH='/usr/local/netcdf/lib:$LD_LIBRARY_PATH' LDFLAGS=-L/usr/local/netcdf/lib ./configure --prefix=/usr/local/netcdf && \
+    rm -rf /var/tmp/v4.7.0.tar.gz /var/tmp/netcdf-c-4.7.0 && \
+    mkdir -p /var/tmp && wget -q -nc --no-check-certificate -P /var/tmp https://github.com/Unidata/netcdf-cxx4/archive/v4.3.0.tar.gz && \
+    mkdir -p /var/tmp && tar -x -f /var/tmp/v4.3.0.tar.gz -C /var/tmp -z && \
+    cd /var/tmp/netcdf-cxx4-4.3.0 &&  CPPFLAGS='-I/usr/local/netcdf/include -I/usr/local/hdf5/include' LD_LIBRARY_PATH='/usr/local/netcdf/lib:$LD_LIBRARY_PATH' LDFLAGS=-L/usr/local/netcdf/lib ./configure --prefix=/usr/local/netcdf && \
     make -j$(nproc) && \
     make -j$(nproc) install && \
-    rm -rf /var/tmp/netcdf-cxx4-4.3.0.tar.gz /var/tmp/netcdf-cxx4-4.3.0 && \
-    mkdir -p /var/tmp && wget -q -nc --no-check-certificate -P /var/tmp ftp://ftp.unidata.ucar.edu/pub/netcdf/netcdf-fortran-4.4.5.tar.gz && \
-    mkdir -p /var/tmp && tar -x -f /var/tmp/netcdf-fortran-4.4.5.tar.gz -C /var/tmp -z && \
-    cd /var/tmp/netcdf-fortran-4.4.5 &&  CPPFLAGS=-I/usr/local/netcdf/include LD_LIBRARY_PATH='/usr/local/netcdf/lib:$LD_LIBRARY_PATH' LDFLAGS=-L/usr/local/netcdf/lib ./configure --prefix=/usr/local/netcdf && \
+    rm -rf /var/tmp/v4.3.0.tar.gz /var/tmp/netcdf-cxx4-4.3.0 && \
+    mkdir -p /var/tmp && wget -q -nc --no-check-certificate -P /var/tmp https://github.com/Unidata/netcdf-fortran/archive/v4.4.5.tar.gz && \
+    mkdir -p /var/tmp && tar -x -f /var/tmp/v4.4.5.tar.gz -C /var/tmp -z && \
+    cd /var/tmp/netcdf-fortran-4.4.5 &&  CPPFLAGS='-I/usr/local/netcdf/include -I/usr/local/hdf5/include' LD_LIBRARY_PATH='/usr/local/netcdf/lib:$LD_LIBRARY_PATH' LDFLAGS=-L/usr/local/netcdf/lib ./configure --prefix=/usr/local/netcdf && \
     make -j$(nproc) && \
     make -j$(nproc) install && \
-    rm -rf /var/tmp/netcdf-fortran-4.4.5.tar.gz /var/tmp/netcdf-fortran-4.4.5
+    rm -rf /var/tmp/v4.4.5.tar.gz /var/tmp/netcdf-fortran-4.4.5
 ENV LD_LIBRARY_PATH=/usr/local/netcdf/lib:$LD_LIBRARY_PATH \
     PATH=/usr/local/netcdf/bin:$PATH''')
 
@@ -87,24 +87,24 @@ RUN yum install -y \
         wget \
         zlib-devel && \
     rm -rf /var/cache/yum/*
-RUN mkdir -p /var/tmp && wget -q -nc --no-check-certificate -P /var/tmp ftp://ftp.unidata.ucar.edu/pub/netcdf/netcdf-c-4.7.0.tar.gz && \
-    mkdir -p /var/tmp && tar -x -f /var/tmp/netcdf-c-4.7.0.tar.gz -C /var/tmp -z && \
+RUN mkdir -p /var/tmp && wget -q -nc --no-check-certificate -P /var/tmp https://github.com/Unidata/netcdf-c/archive/v4.7.0.tar.gz && \
+    mkdir -p /var/tmp && tar -x -f /var/tmp/v4.7.0.tar.gz -C /var/tmp -z && \
     cd /var/tmp/netcdf-c-4.7.0 &&  CPPFLAGS=-I/usr/local/hdf5/include LDFLAGS=-L/usr/local/hdf5/lib ./configure --prefix=/usr/local/netcdf && \
     make -j$(nproc) && \
     make -j$(nproc) install && \
-    rm -rf /var/tmp/netcdf-c-4.7.0.tar.gz /var/tmp/netcdf-c-4.7.0 && \
-    mkdir -p /var/tmp && wget -q -nc --no-check-certificate -P /var/tmp ftp://ftp.unidata.ucar.edu/pub/netcdf/netcdf-cxx4-4.3.0.tar.gz && \
-    mkdir -p /var/tmp && tar -x -f /var/tmp/netcdf-cxx4-4.3.0.tar.gz -C /var/tmp -z && \
-    cd /var/tmp/netcdf-cxx4-4.3.0 &&  CPPFLAGS=-I/usr/local/netcdf/include LD_LIBRARY_PATH='/usr/local/netcdf/lib:$LD_LIBRARY_PATH' LDFLAGS=-L/usr/local/netcdf/lib ./configure --prefix=/usr/local/netcdf && \
+    rm -rf /var/tmp/v4.7.0.tar.gz /var/tmp/netcdf-c-4.7.0 && \
+    mkdir -p /var/tmp && wget -q -nc --no-check-certificate -P /var/tmp https://github.com/Unidata/netcdf-cxx4/archive/v4.3.0.tar.gz && \
+    mkdir -p /var/tmp && tar -x -f /var/tmp/v4.3.0.tar.gz -C /var/tmp -z && \
+    cd /var/tmp/netcdf-cxx4-4.3.0 &&  CPPFLAGS='-I/usr/local/netcdf/include -I/usr/local/hdf5/include' LD_LIBRARY_PATH='/usr/local/netcdf/lib:$LD_LIBRARY_PATH' LDFLAGS=-L/usr/local/netcdf/lib ./configure --prefix=/usr/local/netcdf && \
     make -j$(nproc) && \
     make -j$(nproc) install && \
-    rm -rf /var/tmp/netcdf-cxx4-4.3.0.tar.gz /var/tmp/netcdf-cxx4-4.3.0 && \
-    mkdir -p /var/tmp && wget -q -nc --no-check-certificate -P /var/tmp ftp://ftp.unidata.ucar.edu/pub/netcdf/netcdf-fortran-4.4.5.tar.gz && \
-    mkdir -p /var/tmp && tar -x -f /var/tmp/netcdf-fortran-4.4.5.tar.gz -C /var/tmp -z && \
-    cd /var/tmp/netcdf-fortran-4.4.5 &&  CPPFLAGS=-I/usr/local/netcdf/include LD_LIBRARY_PATH='/usr/local/netcdf/lib:$LD_LIBRARY_PATH' LDFLAGS=-L/usr/local/netcdf/lib ./configure --prefix=/usr/local/netcdf && \
+    rm -rf /var/tmp/v4.3.0.tar.gz /var/tmp/netcdf-cxx4-4.3.0 && \
+    mkdir -p /var/tmp && wget -q -nc --no-check-certificate -P /var/tmp https://github.com/Unidata/netcdf-fortran/archive/v4.4.5.tar.gz && \
+    mkdir -p /var/tmp && tar -x -f /var/tmp/v4.4.5.tar.gz -C /var/tmp -z && \
+    cd /var/tmp/netcdf-fortran-4.4.5 &&  CPPFLAGS='-I/usr/local/netcdf/include -I/usr/local/hdf5/include' LD_LIBRARY_PATH='/usr/local/netcdf/lib:$LD_LIBRARY_PATH' LDFLAGS=-L/usr/local/netcdf/lib ./configure --prefix=/usr/local/netcdf && \
     make -j$(nproc) && \
     make -j$(nproc) install && \
-    rm -rf /var/tmp/netcdf-fortran-4.4.5.tar.gz /var/tmp/netcdf-fortran-4.4.5
+    rm -rf /var/tmp/v4.4.5.tar.gz /var/tmp/netcdf-fortran-4.4.5
 ENV LD_LIBRARY_PATH=/usr/local/netcdf/lib:$LD_LIBRARY_PATH \
     PATH=/usr/local/netcdf/bin:$PATH''')
 
@@ -127,25 +127,25 @@ RUN apt-get update -y && \
         wget \
         zlib1g-dev && \
     rm -rf /var/lib/apt/lists/*
-RUN mkdir -p /var/tmp && wget -q -nc --no-check-certificate -P /var/tmp ftp://ftp.unidata.ucar.edu/pub/netcdf/netcdf-4.6.1.tar.gz && \
-    mkdir -p /var/tmp && tar -x -f /var/tmp/netcdf-4.6.1.tar.gz -C /var/tmp -z && \
-    cd /var/tmp/netcdf-4.6.1 &&  CPPFLAGS=-I/usr/local/hdf5/include LDFLAGS=-L/usr/local/hdf5/lib ./configure --prefix=/usr/local/netcdf && \
+RUN mkdir -p /var/tmp && wget -q -nc --no-check-certificate -P /var/tmp https://github.com/Unidata/netcdf-c/archive/v4.6.1.tar.gz && \
+    mkdir -p /var/tmp && tar -x -f /var/tmp/v4.6.1.tar.gz -C /var/tmp -z && \
+    cd /var/tmp/netcdf-c-4.6.1 &&  CPPFLAGS=-I/usr/local/hdf5/include LDFLAGS=-L/usr/local/hdf5/lib ./configure --prefix=/usr/local/netcdf && \
     make -j$(nproc) && \
     make -j$(nproc) install && \
     echo "/usr/local/netcdf/lib" >> /etc/ld.so.conf.d/hpccm.conf && ldconfig && \
-    rm -rf /var/tmp/netcdf-4.6.1.tar.gz /var/tmp/netcdf-4.6.1 && \
-    mkdir -p /var/tmp && wget -q -nc --no-check-certificate -P /var/tmp ftp://ftp.unidata.ucar.edu/pub/netcdf/netcdf-cxx4-4.3.0.tar.gz && \
-    mkdir -p /var/tmp && tar -x -f /var/tmp/netcdf-cxx4-4.3.0.tar.gz -C /var/tmp -z && \
-    cd /var/tmp/netcdf-cxx4-4.3.0 &&  CPPFLAGS=-I/usr/local/netcdf/include LD_LIBRARY_PATH='/usr/local/netcdf/lib:$LD_LIBRARY_PATH' LDFLAGS=-L/usr/local/netcdf/lib ./configure --prefix=/usr/local/netcdf && \
+    rm -rf /var/tmp/v4.6.1.tar.gz /var/tmp/netcdf-c-4.6.1 && \
+    mkdir -p /var/tmp && wget -q -nc --no-check-certificate -P /var/tmp https://github.com/Unidata/netcdf-cxx4/archive/v4.3.0.tar.gz && \
+    mkdir -p /var/tmp && tar -x -f /var/tmp/v4.3.0.tar.gz -C /var/tmp -z && \
+    cd /var/tmp/netcdf-cxx4-4.3.0 &&  CPPFLAGS='-I/usr/local/netcdf/include -I/usr/local/hdf5/include' LD_LIBRARY_PATH='/usr/local/netcdf/lib:$LD_LIBRARY_PATH' LDFLAGS=-L/usr/local/netcdf/lib ./configure --prefix=/usr/local/netcdf && \
     make -j$(nproc) && \
     make -j$(nproc) install && \
-    rm -rf /var/tmp/netcdf-cxx4-4.3.0.tar.gz /var/tmp/netcdf-cxx4-4.3.0 && \
-    mkdir -p /var/tmp && wget -q -nc --no-check-certificate -P /var/tmp ftp://ftp.unidata.ucar.edu/pub/netcdf/netcdf-fortran-4.4.4.tar.gz && \
-    mkdir -p /var/tmp && tar -x -f /var/tmp/netcdf-fortran-4.4.4.tar.gz -C /var/tmp -z && \
-    cd /var/tmp/netcdf-fortran-4.4.4 &&  CPPFLAGS=-I/usr/local/netcdf/include LD_LIBRARY_PATH='/usr/local/netcdf/lib:$LD_LIBRARY_PATH' LDFLAGS=-L/usr/local/netcdf/lib ./configure --prefix=/usr/local/netcdf && \
+    rm -rf /var/tmp/v4.3.0.tar.gz /var/tmp/netcdf-cxx4-4.3.0 && \
+    mkdir -p /var/tmp && wget -q -nc --no-check-certificate -P /var/tmp https://github.com/Unidata/netcdf-fortran/archive/v4.4.4.tar.gz && \
+    mkdir -p /var/tmp && tar -x -f /var/tmp/v4.4.4.tar.gz -C /var/tmp -z && \
+    cd /var/tmp/netcdf-fortran-4.4.4 &&  CPPFLAGS='-I/usr/local/netcdf/include -I/usr/local/hdf5/include' LD_LIBRARY_PATH='/usr/local/netcdf/lib:$LD_LIBRARY_PATH' LDFLAGS=-L/usr/local/netcdf/lib ./configure --prefix=/usr/local/netcdf && \
     make -j$(nproc) && \
     make -j$(nproc) install && \
-    rm -rf /var/tmp/netcdf-fortran-4.4.4.tar.gz /var/tmp/netcdf-fortran-4.4.4
+    rm -rf /var/tmp/v4.4.4.tar.gz /var/tmp/netcdf-fortran-4.4.4
 ENV PATH=/usr/local/netcdf/bin:$PATH''')
 
     @ubuntu