Repository: NVIDIA/hpc-container-maker

Issue:
nvhpc building block with C++20 library support
According to the [nvhpc docs](https://docs.nvidia.com/hpc-sdk/hpc-sdk-release-notes/index.html#platform-requirements), one needs gcc/glibc>=10.1 for C++20 library support in the new nvhpc 22.3. 

Neither
```
Stage0 += baseimage(image='nvcr.io/nvidia/nvhpc:22.3-devel-cuda11.6-ubuntu20.04')
Stage0 += gnu(version='10')
```
nor 
```
Stage0 += baseimage(image='ubuntu:20.04')
Stage0 += gnu(version='10')
Stage0 += nvhpc(eula=True, cuda_multi=False, version='22.3')
```
seem to do the trick. The second one even actively installs gcc 9 again instead of using the provided gcc 10 toolchain. 
I tested the support by trying to compile a source file with `#include <ranges>` via `nvc++ -std=c++20` inside the container (which works when using the installed `g++` 10.3.0). 

Maybe the nvhpc building block needs a `toolchain` argument?

Discussion:
samcmill: @pauleonix please take a look.  An example recipe is:

```
Stage0 += baseimage(image='ubuntu:20.04')

Stage0 += gnu(version='10')
tc = hpccm.toolchain(CC='gcc-10', CXX='g++-10', F77='gfortran-10')

Stage0 += nvhpc(cuda_multi=False, eula=True, toolchain=tc)
```
pauleonix: Not sure why, but when I try to compile sth (combining standard parallelism, C++20 views and thrust) with this container using
```shell
singularity exec -e test.sif nvc++ -stdpar=gpu -gpu=cc75,cuda11.6 -O3 -std=c++20 -o hpccm_test cpp20par.cpp
```
I get
```shell
"/opt/nvidia/hpc_sdk/Linux_x86_64/22.3/cuda/11.6/include/cuda_runtime_api.h", line 8676: error: identifier "UINT_MAX" is undefined                                         
  extern __CUDA_DEPRECATED __host__ cudaError_t CUDARTAPI cudaBindTexture(size_t *offset, const struct textureReference *texref, const void *devPtr, const struct cudaChannelFormatDesc *desc, size_t size __dv(UINT_
MAX));
                                                                                                                                                                                                           ^

"/opt/nvidia/hpc_sdk/Linux_x86_64/22.3/cuda/11.6/include/cuda_runtime_api.h", line 9257: error: identifier "cudaTextureObject_t" is undefined
  extern __host__ cudaError_t CUDARTAPI cudaCreateTextureObject(cudaTextureObject_t *pTexObject, const struct cudaResourceDesc *pResDesc, const struct cudaTextureDesc *pTexDesc, const struct cudaResourceViewDesc *
pResViewDesc);
                                                                ^

"/opt/nvidia/hpc_sdk/Linux_x86_64/22.3/cuda/11.6/include/cuda_runtime_api.h", line 9257: error: identifier "pTexObject" is undefined
  extern __host__ cudaError_t CUDARTAPI cudaCreateTextureObject(cudaTextureObject_t *pTexObject, const struct cudaResourceDesc *pResDesc, const struct cudaTextureDesc *pTexDesc, const struct cudaResourceViewDesc *
pResViewDesc);
                                                                                     ^

"/opt/nvidia/hpc_sdk/Linux_x86_64/22.3/cuda/11.6/include/cuda_runtime_api.h", line 9257: error: expected an expression
  extern __host__ cudaError_t CUDARTAPI cudaCreateTextureObject(cudaTextureObject_t *pTexObject, const struct cudaResourceDesc *pResDesc, const struct cudaTextureDesc *pTexDesc, const struct cudaResourceViewDesc *
pResViewDesc);
                                                                                                 ^

"/opt/nvidia/hpc_sdk/Linux_x86_64/22.3/cuda/11.6/include/cuda_runtime_api.h", line 9257: error: expected an expression
  extern __host__ cudaError_t CUDARTAPI cudaCreateTextureObject(cudaTextureObject_t *pTexObject, const struct cudaResourceDesc *pResDesc, const struct cudaTextureDesc *pTexDesc, const struct cudaResourceViewDesc *
pResViewDesc);
                                                                                                                                          ^

"/opt/nvidia/hpc_sdk/Linux_x86_64/22.3/cuda/11.6/include/cuda_runtime_api.h", line 9257: error: expected an expression
  extern __host__ cudaError_t CUDARTAPI cudaCreateTextureObject(cudaTextureObject_t *pTexObject, const struct cudaResourceDesc *pResDesc, const struct cudaTextureDesc *pTexDesc, const struct cudaResourceViewDesc *
pResViewDesc);
                                                                                                                                                                                  ^

"/opt/nvidia/hpc_sdk/Linux_x86_64/22.3/cuda/11.6/include/cuda_runtime_api.h", line 9257: error: too many initializer values
  extern __host__ cudaError_t CUDARTAPI cudaCreateTextureObject(cudaTextureObject_t *pTexObject, const struct cudaResourceDesc *pResDesc, const struct cudaTextureDesc *pTexDesc, const struct cudaResourceViewDesc *
pResViewDesc);
                                                                                                                                        ^

"/opt/nvidia/hpc_sdk/Linux_x86_64/22.3/cuda/11.6/include/cuda_runtime_api.h", line 9277: error: expected a ")"
  extern __host__ cudaError_t CUDARTAPI cudaDestroyTextureObject(cudaTextureObject_t texObject);
                                                                                     ^

"/opt/nvidia/hpc_sdk/Linux_x86_64/22.3/cuda/11.6/include/cuda_runtime_api.h", line 9297: error: "cudaTextureObject_t" is not a type name
  extern __host__ cudaError_t CUDARTAPI cudaGetTextureObjectResourceDesc(struct cudaResourceDesc *pResDesc, cudaTextureObject_t texObject);
                                                                                                            ^

"/opt/nvidia/hpc_sdk/Linux_x86_64/22.3/cuda/11.6/include/cuda_runtime_api.h", line 9317: error: "cudaTextureObject_t" is not a type name
  extern __host__ cudaError_t CUDARTAPI cudaGetTextureObjectTextureDesc(struct cudaTextureDesc *pTexDesc, cudaTextureObject_t texObject);
                                                                                                          ^

"/opt/nvidia/hpc_sdk/Linux_x86_64/22.3/cuda/11.6/include/cuda_runtime_api.h", line 9338: error: "cudaTextureObject_t" is not a type name
  extern __host__ cudaError_t CUDARTAPI cudaGetTextureObjectResourceViewDesc(struct cudaResourceViewDesc *pResViewDesc, cudaTextureObject_t texObject);
                                                                                                                        ^

"/opt/nvidia/hpc_sdk/Linux_x86_64/22.3/cuda/11.6/include/cuda_runtime_api.h", line 9383: error: identifier "cudaSurfaceObject_t" is undefined
  extern __host__ cudaError_t CUDARTAPI cudaCreateSurfaceObject(cudaSurfaceObject_t *pSurfObject, const struct cudaResourceDesc *pResDesc);
                                                                ^

"/opt/nvidia/hpc_sdk/Linux_x86_64/22.3/cuda/11.6/include/cuda_runtime_api.h", line 9383: error: identifier "pSurfObject" is undefined
  extern __host__ cudaError_t CUDARTAPI cudaCreateSurfaceObject(cudaSurfaceObject_t *pSurfObject, const struct cudaResourceDesc *pResDesc);
                                                                                     ^

"/opt/nvidia/hpc_sdk/Linux_x86_64/22.3/cuda/11.6/include/cuda_runtime_api.h", line 9383: error: expected an expression
  extern __host__ cudaError_t CUDARTAPI cudaCreateSurfaceObject(cudaSurfaceObject_t *pSurfObject, const struct cudaResourceDesc *pResDesc);
                                                                                                  ^

"/opt/nvidia/hpc_sdk/Linux_x86_64/22.3/cuda/11.6/include/cuda_runtime_api.h", line 9383: error: too many initializer values
  extern __host__ cudaError_t CUDARTAPI cudaCreateSurfaceObject(cudaSurfaceObject_t *pSurfObject, const struct cudaResourceDesc *pResDesc);
                                                                                                                                         ^

"/opt/nvidia/hpc_sdk/Linux_x86_64/22.3/cuda/11.6/include/cuda_runtime_api.h", line 9403: error: expected a ")"
  extern __host__ cudaError_t CUDARTAPI cudaDestroySurfaceObject(cudaSurfaceObject_t surfObject);
                                                                                     ^

"/opt/nvidia/hpc_sdk/Linux_x86_64/22.3/cuda/11.6/include/cuda_runtime_api.h", line 9422: error: "cudaSurfaceObject_t" is not a type name
  extern __host__ cudaError_t CUDARTAPI cudaGetSurfaceObjectResourceDesc(struct cudaResourceDesc *pResDesc, cudaSurfaceObject_t surfObject);
                                                                                                            ^

"/opt/nvidia/hpc_sdk/Linux_x86_64/22.3/cuda/11.6/include/cuda_runtime.h", line 1403: error: identifier "UINT_MAX" is undefined
          size_t                            size = UINT_MAX
                                                   ^

"/opt/nvidia/hpc_sdk/Linux_x86_64/22.3/cuda/11.6/include/cuda_runtime.h", line 1448: error: identifier "UINT_MAX" is undefined
          size_t                            size = UINT_MAX
                                                   ^

"/opt/nvidia/hpc_sdk/Linux_x86_64/22.3/compilers/include-stdpar/thrust/detail/integer_traits.h", line 118: error: identifier "LLONG_MIN" is undefined
        public integer_traits_base<long long, LLONG_MIN, LLONG_MAX>
                                              ^

"/opt/nvidia/hpc_sdk/Linux_x86_64/22.3/compilers/include-stdpar/thrust/detail/integer_traits.h", line 118: error: identifier "LLONG_MAX" is undefined
        public integer_traits_base<long long, LLONG_MIN, LLONG_MAX>
                                                         ^

"/opt/nvidia/hpc_sdk/Linux_x86_64/22.3/compilers/include-stdpar/thrust/detail/integer_traits.h", line 125: error: identifier "ULLONG_MAX" is undefined
        public integer_traits_base<unsigned long long, 0, ULLONG_MAX>
                                                          ^

22 errors detected in the compilation of "cpp20par.cpp".
```
With the previous approach the same compilation works without problems. Any ideas? Should I share `cpp20par.cpp`?
pauleonix: The build looked promising...
```shell
+ /opt/nvidia/hpc_sdk/Linux_x86_64/22.3/compilers/bin/makelocalrc /opt/nvidia/hpc_sdk/Linux_x86_64/22.3/compilers/bin -x -gcc gcc-10 -gpp g++-10 -g77 gfortran-10
+ export CPATH=/opt/nvidia/hpc_sdk/Linux_x86_64/22.3/comm_libs/nvshmem/include:/opt/nvidia/hpc_sdk/Linux_x86_64/22.3/comm_libs/nccl/include:/opt/nvidia/hpc_sdk/Linux_x86_64/22.3/compilers/extras/qd/include/qd:/opt
/nvidia/hpc_sdk/Linux_x86_64/22.3/math_libs/include:/opt/nvidia/hpc_sdk/Linux_x86_64/22.3/comm_libs/mpi/include:
+ export LD_LIBRARY_PATH=/opt/nvidia/hpc_sdk/Linux_x86_64/22.3/comm_libs/nvshmem/lib:/opt/nvidia/hpc_sdk/Linux_x86_64/22.3/comm_libs/nccl/lib:/opt/nvidia/hpc_sdk/Linux_x86_64/22.3/math_libs/lib64:/opt/nvidia/hpc_s
dk/Linux_x86_64/22.3/compilers/lib:/opt/nvidia/hpc_sdk/Linux_x86_64/22.3/cuda/lib64:/opt/nvidia/hpc_sdk/Linux_x86_64/22.3/comm_libs/mpi/lib:/.singularity.d/libs
+ export MANPATH=/opt/nvidia/hpc_sdk/Linux_x86_64/22.3/compilers/man:
+ export PATH=/opt/nvidia/hpc_sdk/Linux_x86_64/22.3/comm_libs/nvshmem/bin:/opt/nvidia/hpc_sdk/Linux_x86_64/22.3/comm_libs/nccl/bin:/opt/nvidia/hpc_sdk/Linux_x86_64/22.3/profilers/bin:/opt/nvidia/hpc_sdk/Linux_x86_
64/22.3/compilers/bin:/opt/nvidia/hpc_sdk/Linux_x86_64/22.3/cuda/bin:/opt/nvidia/hpc_sdk/Linux_x86_64/22.3/comm_libs/mpi/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin
```
pauleonix: When I do `g++ --version` in both images, the working one gives version 10 and the one corresponding to this PR gives version 9. This is probably due to the order of installation? Maybe one needs to make version 10 the default again after installing nvhpc (and version 9 with it)?
pauleonix: Sorry for the noise. It seems to work as intended.
samcmill: Glad to hear this is working for you now.  I was unable to reproduce the issue myself using a trivial stdpar program.

I want to think this through a little more before merging this.  It's confusing that the alternative setup when installing `gcc-10` gets overridden by the `nvhpc` install, and indirectly at that.  This leads to situations like by default `nvcc` uses `-ccbin g++`, which maps to version 9, whereas I think would be reasonable to assume that it should use version 10 by default in this scenario. (One can still manually specify `-ccbin g++-10` of course.)

Getting this to work also required creating a special toolchain, rather than using the toolchain that is automatically created by the `gnu` building block.  E.g., this does not work, although it would be reasonable to expect it to:

```
Stage0 += baseimage(image='ubuntu:20.04')

compiler += gnu(version='10')
Stage0 += compiler

# toolchain.CC = gcc, which maps to gcc-10 before this step, but gets overwritten with gcc-9 as part of the next step
Stage0 += nvhpc(cuda_multi=False, eula=True, toolchain=compiler.toolchain)
```

The only satisfying solution might be to see if the `nvhpc` dependency can be updated to any version of gcc, such that gcc-10 would satisfy it and it wouldn't install `gcc` (`gcc-9`) as a dependency?
pauleonix: Yeah, I also thought that this might need to be fixed in the distributed packages. 

Patch:
diff --git a/docs/building_blocks.md b/docs/building_blocks.md
index a12b606..891b241 100644
--- a/docs/building_blocks.md
+++ b/docs/building_blocks.md
@@ -2863,7 +2863,7 @@ source.  The default value is False.
 use the latest commit on the default branch for the repository.
 
 - __cuda__: Flag to specify the CUDA version of the package to download.
-The default is `11.0`.  This option is ignored if build is True.
+The default is `11.6`.  This option is ignored if build is True.
 
 - __environment__: Boolean flag to specify whether the environment
 (`CPATH`, `LD_LIBRARY_PATH`, `LIBRARY_PATH`, and `PATH`) should be
@@ -2885,7 +2885,7 @@ repository.  The default is empty, i.e., use the release package
 specified by `version`.
 
 - __version__: The version of NCCL to install.  The default value is
-`2.7.8-1`.
+`2.12.10-1`.
 
 __Examples__
 
@@ -3053,7 +3053,7 @@ local build context. The default value is empty.
 
 - __version__: the version of Nsight Compute to install.  Note when
 `runfile` is set this parameter is ignored.  The default value is
-`2020.2.1`.
+`2022.1.1`.
 
 __Examples__
 
@@ -3082,7 +3082,7 @@ __Parameters__
 package should be installed.  The default is True.
 
 - __version__: The version of Nsight Systems to install.  The default
-value is `2021.1.1`.
+value is `2022.2.1`.
 
 __Examples__
 
@@ -3163,6 +3163,10 @@ wildcards are supported.  The default is an empty list.
 be installed by downloading the tar package file.  If False,
 install from the package repository.  The default is False.
 
+- __toolchain__: The toolchain object to be used to configure the HPC
+SDK with a specific GNU toolchain.  The default is empty, i.e., use
+the default GNU toolchain.
+
 - __url__: The location of the package that should be installed.  The default value is `https://developer.download.nvidia.com/hpc-sdk/nvhpc_X_Y_Z_cuda_multi.tar.gz`, where `X, `Y`, and `Z` are the year, version, and architecture whose values are automatically determined.
 
 - __version__: The version of the HPC SDK to use.  Note when `package`
diff --git a/hpccm/building_blocks/nccl.py b/hpccm/building_blocks/nccl.py
index b0e914f..1c8ca17 100644
--- a/hpccm/building_blocks/nccl.py
+++ b/hpccm/building_blocks/nccl.py
@@ -56,7 +56,7 @@ class nccl(bb_base, hpccm.templates.downloader, hpccm.templates.envvars,
     use the latest commit on the default branch for the repository.
 
     cuda: Flag to specify the CUDA version of the package to download.
-    The default is `11.0`.  This option is ignored if build is True.
+    The default is `11.6`.  This option is ignored if build is True.
 
     environment: Boolean flag to specify whether the environment
     (`CPATH`, `LD_LIBRARY_PATH`, `LIBRARY_PATH`, and `PATH`) should be
@@ -78,7 +78,7 @@ class nccl(bb_base, hpccm.templates.downloader, hpccm.templates.envvars,
     specified by `version`.
 
     version: The version of NCCL to install.  The default value is
-    `2.7.8-1`.
+    `2.12.10-1`.
 
     # Examples
 
@@ -102,12 +102,12 @@ class nccl(bb_base, hpccm.templates.downloader, hpccm.templates.envvars,
         self.__build_environment = '' # Filled in by __configure
         self.__default_repository = 'https://github.com/NVIDIA/nccl.git'
         self.__distro_label = ''     # Filled in by __distro
-        self.__cuda = kwargs.pop('cuda', '11.0')
+        self.__cuda = kwargs.pop('cuda', '11.6')
         self.__make_variables = kwargs.pop('make_variables', {})
         self.__ospackages = kwargs.pop('ospackages', [])
         self.__prefix = kwargs.pop('prefix', '/usr/local/nccl')
         self.__src_directory = kwargs.pop('src_directory', None)
-        self.__version = kwargs.pop('version', '2.7.8-1')
+        self.__version = kwargs.pop('version', '2.12.10-1')
         self.__wd = kwargs.get('wd', hpccm.config.g_wd) # working directory
 
         if not self.__build:
@@ -123,13 +123,13 @@ class nccl(bb_base, hpccm.templates.downloader, hpccm.templates.envvars,
                                                    self.__cuda),
                      'libnccl-dev={0}+cuda{1}'.format(self.__version,
                                                       self.__cuda)],
-                apt_keys=['https://developer.download.nvidia.com/compute/machine-learning/repos/{0}/{1}/7fa2af80.pub'.format(self.__distro_label, get_cpu_architecture())],
-                apt_repositories=['deb https://developer.download.nvidia.com/compute/machine-learning/repos/{0}/{1} /'.format(self.__distro_label, get_cpu_architecture())],
+                apt_keys=['https://developer.download.nvidia.com/compute/cuda/repos/{0}/{1}/3bf863cc.pub'.format(self.__distro_label, get_cpu_architecture())],
+                apt_repositories=['deb https://developer.download.nvidia.com/compute/cuda/repos/{0}/{1} /'.format(self.__distro_label, get_cpu_architecture())],
                 yum=['libnccl-{0}+cuda{1}'.format(self.__version, self.__cuda),
                      'libnccl-devel-{0}+cuda{1}'.format(self.__version,
                                                         self.__cuda)],
-                yum_keys=['https://developer.download.nvidia.com/compute/machine-learning/repos/{0}/{1}/7fa2af80.pub'.format(self.__distro_label, get_cpu_architecture())],
-                yum_repositories=['https://developer.download.nvidia.com/compute/machine-learning/repos/{0}/{1}'.format(self.__distro_label, get_cpu_architecture())])
+                yum_keys=['https://developer.download.nvidia.com/compute/cuda/repos/{0}/{1}/3bf863cc.pub'.format(self.__distro_label, get_cpu_architecture())],
+                yum_repositories=['https://developer.download.nvidia.com/compute/cuda/repos/{0}/{1}'.format(self.__distro_label, get_cpu_architecture())])
 
         else:
             # Build from source
@@ -246,10 +246,10 @@ class nccl(bb_base, hpccm.templates.downloader, hpccm.templates.envvars,
             self.rt += packages(
                 apt=['libnccl2={0}+cuda{1}'.format(self.__version,
                                                    self.__cuda)],
-                apt_keys=['https://developer.download.nvidia.com/compute/machine-learning/repos/{0}/{1}/7fa2af80.pub'.format(self.__distro_label, get_cpu_architecture())],
-                apt_repositories=['deb https://developer.download.nvidia.com/compute/machine-learning/repos/{0}/{1} /'.format(self.__distro_label, get_cpu_architecture())],
+                apt_keys=['https://developer.download.nvidia.com/compute/cuda/repos/{0}/{1}/3bf863cc.pub'.format(self.__distro_label, get_cpu_architecture())],
+                apt_repositories=['deb https://developer.download.nvidia.com/compute/cuda/repos/{0}/{1} /'.format(self.__distro_label, get_cpu_architecture())],
                 yum=['libnccl-{0}+cuda{1}'.format(self.__version, self.__cuda)],
-                yum_keys=['https://developer.download.nvidia.com/compute/machine-learning/repos/{0}/{1}/7fa2af80.pub'.format(self.__distro_label, get_cpu_architecture())],
-                yum_repositories=['https://developer.download.nvidia.com/compute/machine-learning/repos/{0}/{1}'.format(self.__distro_label, get_cpu_architecture())])
+                yum_keys=['https://developer.download.nvidia.com/compute/cuda/repos/{0}/{1}/3bf863cc.pub'.format(self.__distro_label, get_cpu_architecture())],
+                yum_repositories=['https://developer.download.nvidia.com/compute/cuda/repos/{0}/{1}'.format(self.__distro_label, get_cpu_architecture())])
 
         return str(self.rt)
diff --git a/hpccm/building_blocks/nsight_compute.py b/hpccm/building_blocks/nsight_compute.py
index c7373e7..7e2a530 100644
--- a/hpccm/building_blocks/nsight_compute.py
+++ b/hpccm/building_blocks/nsight_compute.py
@@ -63,7 +63,7 @@ class nsight_compute(bb_base, hpccm.templates.envvars):
 
     version: the version of Nsight Compute to install.  Note when
     `runfile` is set this parameter is ignored.  The default value is
-    `2020.2.1`.
+    `2022.1.1`.
 
     # Examples
 
@@ -89,7 +89,7 @@ class nsight_compute(bb_base, hpccm.templates.envvars):
         self.__prefix = kwargs.get('prefix',
                                    '/usr/local/NVIDIA-Nsight-Compute')
         self.__runfile = kwargs.get('runfile', None)
-        self.__version = kwargs.get('version', '2020.2.1')
+        self.__version = kwargs.get('version', '2022.1.1')
         self.__wd = kwargs.get('wd', posixpath.join(
             hpccm.config.g_wd, 'nsight_compute')) # working directory
 
diff --git a/hpccm/building_blocks/nsight_systems.py b/hpccm/building_blocks/nsight_systems.py
index 4a89e99..5023142 100644
--- a/hpccm/building_blocks/nsight_systems.py
+++ b/hpccm/building_blocks/nsight_systems.py
@@ -41,7 +41,7 @@ class nsight_systems(bb_base):
     package should be installed.  The default is True.
 
     version: The version of Nsight Systems to install.  The default
-    value is `2021.1.1`.
+    value is `2022.2.1`.
 
     # Examples
 
@@ -60,7 +60,7 @@ class nsight_systems(bb_base):
         self.__cli = kwargs.get('cli', True)
         self.__distro_label = ''     # Filled in by __distro
         self.__ospackages = kwargs.get('ospackages', [])
-        self.__version = kwargs.get('version', '2021.1.1')
+        self.__version = kwargs.get('version', '2022.2.1')
 
         # Set the CPU architecture specific parameters
         self.__cpu_arch()
diff --git a/hpccm/building_blocks/nvhpc.py b/hpccm/building_blocks/nvhpc.py
index ed18aab..c1bd17a 100644
--- a/hpccm/building_blocks/nvhpc.py
+++ b/hpccm/building_blocks/nvhpc.py
@@ -107,6 +107,10 @@ class nvhpc(bb_base, hpccm.templates.downloader, hpccm.templates.envvars,
     be installed by downloading the tar package file.  If False,
     install from the package repository.  The default is False.
 
+    toolchain: The toolchain object to be used to configure the HPC
+    SDK with a specific GNU toolchain.  The default is empty, i.e., use
+    the default GNU toolchain.
+
     url: The location of the package that should be installed.  The default value is `https://developer.download.nvidia.com/hpc-sdk/nvhpc_X_Y_Z_cuda_multi.tar.gz`, where `X, `Y`, and `Z` are the year, version, and architecture whose values are automatically determined.
 
     version: The version of the HPC SDK to use.  Note when `package`
@@ -167,6 +171,7 @@ class nvhpc(bb_base, hpccm.templates.downloader, hpccm.templates.envvars,
         self.__redist = kwargs.get('redist', [])
         self.__stdpar_cudacc = kwargs.get('stdpar_cudacc', None)
         self.__tarball = kwargs.get('tarball', False)
+        self.__toolchain = kwargs.get('toolchain', None)
         self.__url = kwargs.get('url', None)
         self.__version = kwargs.get('version', '22.3')
         self.__wd = kwargs.get('wd', hpccm.config.g_wd) # working directory
@@ -237,6 +242,23 @@ class nvhpc(bb_base, hpccm.templates.downloader, hpccm.templates.envvars,
                 ospackages=[self.__nvhpc_package],
                 yum_repositories=['https://developer.download.nvidia.com/hpc-sdk/rhel/nvhpc.repo'])
 
+        if self.__toolchain:
+            # Regenerate the localrc using the compilers from the specified
+            # toolchain
+            compiler_bin = posixpath.join(self.__basepath, 'compilers', 'bin')
+
+            args = ['-x']
+            if self.__toolchain.CC:
+                args.append('-gcc {}'.format(self.__toolchain.CC))
+            if self.__toolchain.CXX:
+                args.append('-gpp {}'.format(self.__toolchain.CXX))
+            if self.__toolchain.F77:
+                args.append('-g77 {}'.format(self.__toolchain.F77))
+
+            self += shell(commands=['{0} {1} {2}'.format(
+                posixpath.join(compiler_bin, 'makelocalrc'), compiler_bin,
+                ' '.join(args))])
+
         self += environment(variables=self.environment_step())
 
     def __cpu_arch(self):


Test patch:
diff --git a/test/test_nccl.py b/test/test_nccl.py
index 7d85c5a..df23c72 100644
--- a/test/test_nccl.py
+++ b/test/test_nccl.py
@@ -38,7 +38,7 @@ class Test_nccl(unittest.TestCase):
         """nccl defaults"""
         n = nccl()
         self.assertEqual(str(n),
-r'''# NCCL 2.7.8-1
+r'''# NCCL 2.12.10-1
 RUN apt-get update -y && \
     DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends \
         apt-transport-https \
@@ -46,12 +46,12 @@ RUN apt-get update -y && \
         gnupg \
         wget && \
     rm -rf /var/lib/apt/lists/*
-RUN wget -qO - https://developer.download.nvidia.com/compute/machine-learning/repos/ubuntu1604/x86_64/7fa2af80.pub | apt-key add - && \
-    echo "deb https://developer.download.nvidia.com/compute/machine-learning/repos/ubuntu1604/x86_64 /" >> /etc/apt/sources.list.d/hpccm.list && \
+RUN wget -qO - https://developer.download.nvidia.com/compute/cuda/repos/ubuntu1604/x86_64/3bf863cc.pub | apt-key add - && \
+    echo "deb https://developer.download.nvidia.com/compute/cuda/repos/ubuntu1604/x86_64 /" >> /etc/apt/sources.list.d/hpccm.list && \
     apt-get update -y && \
     DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends \
-        libnccl-dev=2.7.8-1+cuda11.0 \
-        libnccl2=2.7.8-1+cuda11.0 && \
+        libnccl-dev=2.12.10-1+cuda11.6 \
+        libnccl2=2.12.10-1+cuda11.6 && \
     rm -rf /var/lib/apt/lists/*''')
 
     @x86_64
@@ -61,7 +61,7 @@ RUN wget -qO - https://developer.download.nvidia.com/compute/machine-learning/re
         """nccl defaults"""
         n = nccl()
         self.assertEqual(str(n),
-r'''# NCCL 2.7.8-1
+r'''# NCCL 2.12.10-1
 RUN apt-get update -y && \
     DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends \
         apt-transport-https \
@@ -69,12 +69,12 @@ RUN apt-get update -y && \
         gnupg \
         wget && \
     rm -rf /var/lib/apt/lists/*
-RUN wget -qO - https://developer.download.nvidia.com/compute/machine-learning/repos/ubuntu1804/x86_64/7fa2af80.pub | apt-key add - && \
-    echo "deb https://developer.download.nvidia.com/compute/machine-learning/repos/ubuntu1804/x86_64 /" >> /etc/apt/sources.list.d/hpccm.list && \
+RUN wget -qO - https://developer.download.nvidia.com/compute/cuda/repos/ubuntu1804/x86_64/3bf863cc.pub | apt-key add - && \
+    echo "deb https://developer.download.nvidia.com/compute/cuda/repos/ubuntu1804/x86_64 /" >> /etc/apt/sources.list.d/hpccm.list && \
     apt-get update -y && \
     DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends \
-        libnccl-dev=2.7.8-1+cuda11.0 \
-        libnccl2=2.7.8-1+cuda11.0 && \
+        libnccl-dev=2.12.10-1+cuda11.6 \
+        libnccl2=2.12.10-1+cuda11.6 && \
     rm -rf /var/lib/apt/lists/*''')
 
     @ppc64le
@@ -92,8 +92,8 @@ RUN apt-get update -y && \
         gnupg \
         wget && \
     rm -rf /var/lib/apt/lists/*
-RUN wget -qO - https://developer.download.nvidia.com/compute/machine-learning/repos/ubuntu1604/ppc64el/7fa2af80.pub | apt-key add - && \
-    echo "deb https://developer.download.nvidia.com/compute/machine-learning/repos/ubuntu1604/ppc64el /" >> /etc/apt/sources.list.d/hpccm.list && \
+RUN wget -qO - https://developer.download.nvidia.com/compute/cuda/repos/ubuntu1604/ppc64el/3bf863cc.pub | apt-key add - && \
+    echo "deb https://developer.download.nvidia.com/compute/cuda/repos/ubuntu1604/ppc64el /" >> /etc/apt/sources.list.d/hpccm.list && \
     apt-get update -y && \
     DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends \
         libnccl-dev=2.4.8-1+cuda9.2 \
@@ -113,11 +113,11 @@ RUN apt-get update -y && \
         make \
         wget && \
     rm -rf /var/lib/apt/lists/*
-RUN mkdir -p /var/tmp && wget -q -nc --no-check-certificate -P /var/tmp https://github.com/NVIDIA/nccl/archive/v2.7.8-1.tar.gz && \
-    mkdir -p /var/tmp && tar -x -f /var/tmp/v2.7.8-1.tar.gz -C /var/tmp -z && \
-    cd /var/tmp/nccl-2.7.8-1 && \
+RUN mkdir -p /var/tmp && wget -q -nc --no-check-certificate -P /var/tmp https://github.com/NVIDIA/nccl/archive/v2.12.10-1.tar.gz && \
+    mkdir -p /var/tmp && tar -x -f /var/tmp/v2.12.10-1.tar.gz -C /var/tmp -z && \
+    cd /var/tmp/nccl-2.12.10-1 && \
     PREFIX=/usr/local/nccl make -j$(nproc) install && \
-    rm -rf /var/tmp/nccl-2.7.8-1 /var/tmp/v2.7.8-1.tar.gz
+    rm -rf /var/tmp/nccl-2.12.10-1 /var/tmp/v2.12.10-1.tar.gz
 ENV CPATH=/usr/local/nccl/include:$CPATH \
     LD_LIBRARY_PATH=/usr/local/nccl/lib:$LD_LIBRARY_PATH \
     LIBRARY_PATH=/usr/local/nccl/lib:$LIBRARY_PATH \
@@ -130,13 +130,13 @@ ENV CPATH=/usr/local/nccl/include:$CPATH \
         """nccl defaults"""
         n = nccl()
         self.assertEqual(str(n),
-r'''# NCCL 2.7.8-1
-RUN rpm --import https://developer.download.nvidia.com/compute/machine-learning/repos/rhel7/x86_64/7fa2af80.pub && \
+r'''# NCCL 2.12.10-1
+RUN rpm --import https://developer.download.nvidia.com/compute/cuda/repos/rhel7/x86_64/3bf863cc.pub && \
     yum install -y yum-utils && \
-    yum-config-manager --add-repo https://developer.download.nvidia.com/compute/machine-learning/repos/rhel7/x86_64 && \
+    yum-config-manager --add-repo https://developer.download.nvidia.com/compute/cuda/repos/rhel7/x86_64 && \
     yum install -y \
-        libnccl-2.7.8-1+cuda11.0 \
-        libnccl-devel-2.7.8-1+cuda11.0 && \
+        libnccl-2.12.10-1+cuda11.6 \
+        libnccl-devel-2.12.10-1+cuda11.6 && \
     rm -rf /var/cache/yum/*''')
 
     @x86_64
@@ -146,13 +146,13 @@ RUN rpm --import https://developer.download.nvidia.com/compute/machine-learning/
         """nccl defaults"""
         n = nccl()
         self.assertEqual(str(n),
-r'''# NCCL 2.7.8-1
-RUN rpm --import https://developer.download.nvidia.com/compute/machine-learning/repos/rhel8/x86_64/7fa2af80.pub && \
+r'''# NCCL 2.12.10-1
+RUN rpm --import https://developer.download.nvidia.com/compute/cuda/repos/rhel8/x86_64/3bf863cc.pub && \
     yum install -y dnf-utils && \
-    yum-config-manager --add-repo https://developer.download.nvidia.com/compute/machine-learning/repos/rhel8/x86_64 && \
+    yum-config-manager --add-repo https://developer.download.nvidia.com/compute/cuda/repos/rhel8/x86_64 && \
     yum install -y \
-        libnccl-2.7.8-1+cuda11.0 \
-        libnccl-devel-2.7.8-1+cuda11.0 && \
+        libnccl-2.12.10-1+cuda11.6 \
+        libnccl-devel-2.12.10-1+cuda11.6 && \
     rm -rf /var/cache/yum/*''')
 
     @x86_64
@@ -217,11 +217,11 @@ RUN apt-get update -y && \
         gnupg \
         wget && \
     rm -rf /var/lib/apt/lists/*
-RUN wget -qO - https://developer.download.nvidia.com/compute/machine-learning/repos/ubuntu1604/x86_64/7fa2af80.pub | apt-key add - && \
-    echo "deb https://developer.download.nvidia.com/compute/machine-learning/repos/ubuntu1604/x86_64 /" >> /etc/apt/sources.list.d/hpccm.list && \
+RUN wget -qO - https://developer.download.nvidia.com/compute/cuda/repos/ubuntu1604/x86_64/3bf863cc.pub | apt-key add - && \
+    echo "deb https://developer.download.nvidia.com/compute/cuda/repos/ubuntu1604/x86_64 /" >> /etc/apt/sources.list.d/hpccm.list && \
     apt-get update -y && \
     DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends \
-        libnccl2=2.7.8-1+cuda11.0 && \
+        libnccl2=2.12.10-1+cuda11.6 && \
     rm -rf /var/lib/apt/lists/*''')
 
     @x86_64
diff --git a/test/test_nsight_compute.py b/test/test_nsight_compute.py
index 6365610..5b3c3b9 100644
--- a/test/test_nsight_compute.py
+++ b/test/test_nsight_compute.py
@@ -38,7 +38,7 @@ class Test_nsight_compute(unittest.TestCase):
         """Default nsight_compute building block"""
         n = nsight_compute()
         self.assertEqual(str(n),
-r'''# NVIDIA Nsight Compute 2020.2.1
+r'''# NVIDIA Nsight Compute 2022.1.1
 RUN apt-get update -y && \
     DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends \
         apt-transport-https \
@@ -50,10 +50,10 @@ RUN wget -qO - https://developer.download.nvidia.com/devtools/repos/ubuntu1604/a
     echo "deb https://developer.download.nvidia.com/devtools/repos/ubuntu1604/amd64/ /" >> /etc/apt/sources.list.d/hpccm.list && \
     apt-get update -y && \
     DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends \
-        nsight-compute-2020.2.1 && \
+        nsight-compute-2022.1.1 && \
     rm -rf /var/lib/apt/lists/*
 ENV NV_COMPUTE_PROFILER_DISABLE_STOCK_FILE_DEPLOYMENT=1 \
-    PATH=/opt/nvidia/nsight-compute/2020.2.1:$PATH''')
+    PATH=/opt/nvidia/nsight-compute/2022.1.1:$PATH''')
 
     @x86_64
     @centos8
@@ -62,15 +62,15 @@ ENV NV_COMPUTE_PROFILER_DISABLE_STOCK_FILE_DEPLOYMENT=1 \
         """Default nsight_compute building block"""
         n = nsight_compute()
         self.assertEqual(str(n),
-r'''# NVIDIA Nsight Compute 2020.2.1
+r'''# NVIDIA Nsight Compute 2022.1.1
 RUN rpm --import https://developer.download.nvidia.com/devtools/repos/rhel8/x86_64/nvidia.pub && \
     yum install -y dnf-utils && \
     (yum-config-manager --add-repo https://developer.download.nvidia.com/devtools/repos/rhel8/x86_64 || true) && \
     yum install -y \
-        nsight-compute-2020.2.1 && \
+        nsight-compute-2022.1.1 && \
     rm -rf /var/cache/yum/*
 ENV NV_COMPUTE_PROFILER_DISABLE_STOCK_FILE_DEPLOYMENT=1 \
-    PATH=/opt/nvidia/nsight-compute/2020.2.1:$PATH''')
+    PATH=/opt/nvidia/nsight-compute/2022.1.1:$PATH''')
 
     @x86_64
     @ubuntu
diff --git a/test/test_nsight_systems.py b/test/test_nsight_systems.py
index 2d3d128..8c3012e 100644
--- a/test/test_nsight_systems.py
+++ b/test/test_nsight_systems.py
@@ -38,7 +38,7 @@ class Test_nsight_systems(unittest.TestCase):
         """Default nsight_systems building block"""
         n = nsight_systems()
         self.assertEqual(str(n),
-r'''# NVIDIA Nsight Systems 2021.1.1
+r'''# NVIDIA Nsight Systems 2022.2.1
 RUN apt-get update -y && \
     DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends \
         apt-transport-https \
@@ -50,7 +50,7 @@ RUN wget -qO - https://developer.download.nvidia.com/devtools/repos/ubuntu1604/a
     echo "deb https://developer.download.nvidia.com/devtools/repos/ubuntu1604/amd64/ /" >> /etc/apt/sources.list.d/hpccm.list && \
     apt-get update -y && \
     DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends \
-        nsight-systems-cli-2021.1.1 && \
+        nsight-systems-cli-2022.2.1 && \
     rm -rf /var/lib/apt/lists/*''')
 
     @x86_64
@@ -60,12 +60,12 @@ RUN wget -qO - https://developer.download.nvidia.com/devtools/repos/ubuntu1604/a
         """Default nsight_systems building block"""
         n = nsight_systems()
         self.assertEqual(str(n),
-r'''# NVIDIA Nsight Systems 2021.1.1
+r'''# NVIDIA Nsight Systems 2022.2.1
 RUN rpm --import https://developer.download.nvidia.com/devtools/repos/rhel8/x86_64/nvidia.pub && \
     yum install -y dnf-utils && \
     (yum-config-manager --add-repo https://developer.download.nvidia.com/devtools/repos/rhel8/x86_64 || true) && \
     yum install -y \
-        nsight-systems-cli-2021.1.1 && \
+        nsight-systems-cli-2022.2.1 && \
     rm -rf /var/cache/yum/*''')
 
     @x86_64
diff --git a/test/test_nvhpc.py b/test/test_nvhpc.py
index e855cc6..138901c 100644
--- a/test/test_nvhpc.py
+++ b/test/test_nvhpc.py
@@ -25,6 +25,7 @@ import unittest
 from helpers import aarch64, centos, docker, ppc64le, ubuntu, x86_64
 
 from hpccm.building_blocks.nvhpc import nvhpc
+from hpccm.toolchain import toolchain
 
 class Test_nvhpc(unittest.TestCase):
     def setUp(self):
@@ -241,6 +242,30 @@ COPY --from=0 /opt/nvidia/hpc_sdk/Linux_x86_64/22.3/REDIST/math_libs/11.0/lib64/
 COPY --from=0 /opt/nvidia/hpc_sdk/Linux_x86_64/22.3/REDIST/math_libs/11.0/lib64/libcublas.so.11 /opt/nvidia/hpc_sdk/Linux_x86_64/22.3/math_libs/11.0/lib64/libcublas.so.11
 ENV LD_LIBRARY_PATH=/opt/nvidia/hpc_sdk/Linux_x86_64/22.3/comm_libs/11.0/nccl/lib:/opt/nvidia/hpc_sdk/Linux_x86_64/22.3/compilers/lib:/opt/nvidia/hpc_sdk/Linux_x86_64/22.3/math_libs/11.0/lib64:$LD_LIBRARY_PATH''')
 
+    @x86_64
+    @ubuntu
+    @docker
+    def test_makelocalrc(self):
+        """makelocalrc"""
+        tc = toolchain(CC='gcc-10', CXX='g++-10', F77='gfortran-10')
+        n = nvhpc(eula=True, mpi=False, toolchain=tc, version='22.3')
+        self.assertMultiLineEqual(str(n),
+r'''# NVIDIA HPC SDK version 22.3
+RUN apt-get update -y && \
+    DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends \
+        ca-certificates && \
+    rm -rf /var/lib/apt/lists/*
+RUN echo "deb [trusted=yes] https://developer.download.nvidia.com/hpc-sdk/ubuntu/amd64 /" >> /etc/apt/sources.list.d/hpccm.list && \
+    apt-get update -y && \
+    DEBIAN_FRONTEND=noninteractive apt-get install -y --no-install-recommends \
+        nvhpc-22-3-cuda-multi && \
+    rm -rf /var/lib/apt/lists/*
+RUN /opt/nvidia/hpc_sdk/Linux_x86_64/22.3/compilers/bin/makelocalrc /opt/nvidia/hpc_sdk/Linux_x86_64/22.3/compilers/bin -x -gcc gcc-10 -gpp g++-10 -g77 gfortran-10
+ENV CPATH=/opt/nvidia/hpc_sdk/Linux_x86_64/22.3/comm_libs/nvshmem/include:/opt/nvidia/hpc_sdk/Linux_x86_64/22.3/comm_libs/nccl/include:/opt/nvidia/hpc_sdk/Linux_x86_64/22.3/compilers/extras/qd/include/qd:/opt/nvidia/hpc_sdk/Linux_x86_64/22.3/math_libs/include:$CPATH \
+    LD_LIBRARY_PATH=/opt/nvidia/hpc_sdk/Linux_x86_64/22.3/comm_libs/nvshmem/lib:/opt/nvidia/hpc_sdk/Linux_x86_64/22.3/comm_libs/nccl/lib:/opt/nvidia/hpc_sdk/Linux_x86_64/22.3/math_libs/lib64:/opt/nvidia/hpc_sdk/Linux_x86_64/22.3/compilers/lib:/opt/nvidia/hpc_sdk/Linux_x86_64/22.3/cuda/lib64:$LD_LIBRARY_PATH \
+    MANPATH=/opt/nvidia/hpc_sdk/Linux_x86_64/22.3/compilers/man:$MANPATH \
+    PATH=/opt/nvidia/hpc_sdk/Linux_x86_64/22.3/comm_libs/nvshmem/bin:/opt/nvidia/hpc_sdk/Linux_x86_64/22.3/comm_libs/nccl/bin:/opt/nvidia/hpc_sdk/Linux_x86_64/22.3/profilers/bin:/opt/nvidia/hpc_sdk/Linux_x86_64/22.3/compilers/bin:/opt/nvidia/hpc_sdk/Linux_x86_64/22.3/cuda/bin:$PATH''')
+
     def test_toolchain(self):
         """Toolchain"""
         n = nvhpc()