Repository: celeritas-project/celeritas
Pull request #1669: Default downstream CUDA/HIP architectures
Author: login: sethrj
Labels: name: enhancement
description: New feature or request
name: core
description: Software engineering infrastructure (corecel)
Created: 2025-03-14T12:44:02Z | Merged: 2025-03-14T20:36:58Z

If a user builds an app downstream from Celeritas without specifying CUDA/HIP "components", they may not have set the cmake/hip architectures necessary for compatibility.  This PR uses the CeleritasConfig file to set default values before the CUDA/HIP languages are activated. If the user has already declared CUDA/HIP as a language, the `CMAKE_{lang}_ARCHITECTURES` will have already been set by CMake and the Celeritas ones will not be loaded.

When loaded externally, we also now print statements about what variables are being set. Cache variables are printed only on the first configure with a verbosity of "STATUS", and local variables are printed with VERBOSE, matching our other CMake macros.

Also the CI wasn't building the "offload" example for docker because of how the Geant4 version was defined in the CI. This enables `accel` example testing for CUDA/HIP.

These changes are pulled from #1664.


Diff:
diff --git a/.github/workflows/build-spack.yml b/.github/workflows/build-spack.yml
index ac459d2615..ea7ba4c6a5 100644
--- a/.github/workflows/build-spack.yml
+++ b/.github/workflows/build-spack.yml
@@ -280,7 +280,6 @@ jobs:
                || !matrix.geant
               ) && '1' || ''
             }}
-          CELER_GEANT4_VERSION: ${{matrix.geant}}
         if: >-
           ${{
             (matrix.special != 'asanlite')
diff --git a/CMakeLists.txt b/CMakeLists.txt
index 2d730e5545..7b6fd16013 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -713,13 +713,11 @@ foreach(_key
     set(_cache_val "$CACHE{${_key}}")
     if(_cache_val)
       list(APPEND CELERITAS_EXPORT_VARIABLES
-        "set(${_key} \"${_cache_val}\" CACHE PATH \"Set by CeleritasConfig.cmake\")"
+        "_celer_set_cache_default(${_key} \"${_cache_val}\" PATH \"Set by CeleritasConfig.cmake\")"
       )
     else()
     list(APPEND CELERITAS_EXPORT_VARIABLES
-      "if(NOT DEFINED ${_key})"
-      "  set(${_key} \"${_val}\")"
-      "endif()"
+      "_celer_set_local_default(${_key} \"${_val}\")"
     )
   endif()
   endif()
diff --git a/cmake/CeleritasConfig.cmake.in b/cmake/CeleritasConfig.cmake.in
index 19466e6fb8..6a44202e45 100644
--- a/cmake/CeleritasConfig.cmake.in
+++ b/cmake/CeleritasConfig.cmake.in
@@ -29,6 +29,24 @@ if(((@PROJECT_NAME@_FIND_VERSION_MAJOR EQUAL 0
    )
 endif()
 
+#-----------------------------------------------------------------------------#
+# Configuration macros
+#-----------------------------------------------------------------------------#
+
+macro(_celer_set_local_default var val)
+  if(NOT DEFINED ${var})
+    message(VERBOSE "Celeritas: set ${var}=${val}")
+    set(${var} "${val}")
+  endif()
+endmacro()
+
+macro(_celer_set_cache_default var val)
+  if(NOT DEFINED ${var})
+    message(STATUS "Celeritas: set cache ${var}=${val}")
+    set(${var} "${val}" CACHE ${ARGN})
+  endif()
+endmacro()
+
 #-----------------------------------------------------------------------------#
 # Variables
 #-----------------------------------------------------------------------------#
@@ -46,6 +64,13 @@ include(CMakeFindDependencyMacro)
 if(CELERITAS_USE_CUDA)
   find_dependency(CUDAToolkit REQUIRED)
 
+  # Set cache variable if needed *before* checking CUDA
+  _celer_set_cache_default(
+    CMAKE_CUDA_ARCHITECTURES
+    "${CELERITAS_CMAKE_CUDA_ARCHITECTURES}"
+    STRING "CUDA architectures"
+  )
+
   if(NOT CMAKE_CUDA_COMPILER_LOADED)
     # enable_language only really works in top-level CMake directory, not even subprojects
     # https://discourse.cmake.org/t/is-it-appropriate-to-use-enable-language-in-a-cmake-package-file/4335
@@ -65,6 +90,11 @@ if(CELERITAS_USE_CUDA)
     endif()
   endif()
 elseif(CELERITAS_USE_HIP)
+  _celer_set_cache_default(
+    CMAKE_HIP_ARCHITECTURES
+    "${CELERITAS_CMAKE_HIP_ARCHITECTURES}"
+    STRING "HIP architectures"
+  )
   find_dependency(hip REQUIRED QUIET)
 endif()
 
diff --git a/scripts/ci/test-examples.sh b/scripts/ci/test-examples.sh
index 8b2b64c8c8..a1ede940a2 100755
--- a/scripts/ci/test-examples.sh
+++ b/scripts/ci/test-examples.sh
@@ -41,17 +41,26 @@ build_local
 
 # Run Geant4 app examples
 if [ -z "${CELER_DISABLE_ACCEL_EXAMPLES}" ]; then
+  if [ -z "${G4VERSION_NUMBER}" ]; then
+    # Get the geant4 version 11.2.3, failing if config isn't found
+    _vers=$(geant4-config --version)
+    # Replace . with ' ' and convert to MMmp (major/minor/patch)
+    G4VERSION_NUMBER=$(echo "${_vers}" | tr '.' ' ' | xargs printf '%d%01d%01d')
+    echo "Set G4VERSION_NUMBER=\"${G4VERSION_NUMBER}\""
+  fi
+
   # Run small accel examples
   cd "${CELER_SOURCE_DIR}/example/accel"
   build_local
   ctest -V --no-tests=error
 
-  if [ "$(echo $CELER_GEANT4_VERSION | awk -F. '{print $1*100 + $2}')" \
-      -ge 1100 ]; then
+  if [ "${G4VERSION_NUMBER}" -ge 1100 ]; then
     # Run offload-template only on Geant4 v11
     cd "${CELER_SOURCE_DIR}/example/offload-template"
     build_local
-    ./run-offload
+    CELER_DISABLE_PARALLEL=1 \
+      G4FORCENUMBEROFTHREADS=4 G4RUN_MANAGER_TYPE=MT \
+      ./run-offload
   fi
 else
   printf "\e[31mSkipping 'accel' tests due to insufficient requirements\e[m\n"


Reviews:
user: login: pcanal
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-03-14T13:07:44Z
user: login: sethrj
state: COMMENTED
author_association: MEMBER
submitted_at: 2025-03-14T13:34:27Z
user: login: pcanal
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-03-14T17:11:07Z
user: login: pcanal
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-03-14T17:12:13Z
user: login: pcanal
body: LGTM (slight doc update needed)
state: APPROVED
author_association: CONTRIBUTOR
submitted_at: 2025-03-14T18:16:35Z
user: login: pcanal
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-03-14T18:16:53Z

Review comments:
diff_hunk: @@ -96,6 +96,12 @@ jobs:
           }}
         run: |
           . /etc/profile
+          # Get Geant4 version number for use by script
path: .github/workflows/build-docker.yml
user: login: pcanal
body: Other options includes:
```
export G4VERSION_NUMBER=`/opt/view/bin/geant4-config --version | awk -F. '{print $1*100 + $2*10}')"`
```
and (at least in the ATLAS environment):
```
export G4VERSION_NUMBER=`echo $G4VERS  | awk -F. '{print $2*100 + $3*10}')"`
```
albeit the second is probably 'too much' information:
```
$ geant4-config --version
11.1.3
$ echo $G4VERS
geant4.11.1.patch03.atlasmt02
```
created_at: 2025-03-14T13:07:44Z
author_association: CONTRIBUTOR
diff_hunk: @@ -96,6 +96,12 @@ jobs:
           }}
         run: |
           . /etc/profile
+          # Get Geant4 version number for use by script
path: .github/workflows/build-docker.yml
user: login: sethrj
body: Neat! didn't know that trick.
created_at: 2025-03-14T13:34:27Z
author_association: MEMBER
diff_hunk: @@ -41,17 +43,25 @@ build_local
 
 # Run Geant4 app examples
 if [ -z "${CELER_DISABLE_ACCEL_EXAMPLES}" ]; then
+  if [ -z "${G4VERSION_NUMBER}" ]; then
+    # Get the geant4 version 11.2.3, replace `.` with ` `, concat as 110203
+    _vers=$(geant4-config --version | tr '.' ' ')
+    G4VERSION_NUMBER=$(printf '%d%02d%02d' ${_vers})
path: scripts/ci/test-examples.sh
user: login: pcanal
body: I don't know that it matters but the `G4Version.h` claims that the 2nd and 3rd numbers are restricted to a single digit and write the version down as `1123`. I.e.:
```suggestion
    # Get the geant4 version 11.2.3, replace `.` with ` `, concat as 110203
    _vers=$(geant4-config --version | tr '.' ' ')
    G4VERSION_NUMBER=$(printf '%d%01d%01d' ${_vers})
```
created_at: 2025-03-14T17:11:07Z
author_association: CONTRIBUTOR
diff_hunk: @@ -41,17 +43,25 @@ build_local
 
 # Run Geant4 app examples
 if [ -z "${CELER_DISABLE_ACCEL_EXAMPLES}" ]; then
+  if [ -z "${G4VERSION_NUMBER}" ]; then
+    # Get the geant4 version 11.2.3, replace `.` with ` `, concat as 110203
+    _vers=$(geant4-config --version | tr '.' ' ')
+    G4VERSION_NUMBER=$(printf '%d%02d%02d' ${_vers})
+    echo "Set G4VERSION_NUMBER=${G4VERSION_NUMBER}"
+  fi
+
   # Run small accel examples
   cd "${CELER_SOURCE_DIR}/example/accel"
   build_local
   ctest -V --no-tests=error
 
-  if [ "$(echo $CELER_GEANT4_VERSION | awk -F. '{print $1*100 + $2}')" \
-      -ge 1100 ]; then
+  if [ "${G4VERSION_NUMBER}" -ge 1100 ]; then
path: scripts/ci/test-examples.sh
user: login: pcanal
body: This fails with the current code at https://github.com/celeritas-project/celeritas/pull/1669/files#r1995958825
but would work with the proposed updated.  Alternatively:
```suggestion
  if [ "${G4VERSION_NUMBER}" -ge 110000 ]; then
```
created_at: 2025-03-14T17:12:13Z
author_association: CONTRIBUTOR
diff_hunk: @@ -41,17 +43,25 @@ build_local
 
 # Run Geant4 app examples
 if [ -z "${CELER_DISABLE_ACCEL_EXAMPLES}" ]; then
+  if [ -z "${G4VERSION_NUMBER}" ]; then
+    # Get the geant4 version 11.2.3, replace `.` with ` `, concat as 110203
path: scripts/ci/test-examples.sh
user: login: pcanal
body: ```suggestion
    # Get the geant4 version 11.2.3, replace `.` with ` `, concat as 1123
```
created_at: 2025-03-14T18:16:53Z
author_association: CONTRIBUTOR

PR comments:
user: login: github-actions[bot]
created_at: 2025-03-14T13:14:09Z
body: ## Test summary
 4 557 files   6 961 suites   14m 14s :stopwatch:
 1 775 tests  1 764 :white_check_mark: 11 :zzz: 0 :x:
24 152 runs  24 064 :white_check_mark: 88 :zzz: 0 :x:

Results for commit fb0d9f6f.

[test-results]:data:application/gzip;base64,H4sIACqL1GcC/03MyQ6DIBSF4VcxrLtguoB9mQaRm5A6NAgr03cv4lCX5zvJvxIMg1/Is5EA+tGQJYdUt2oVK7vP0aYwT0UMyALlTdvPtIZzvpbsXDUl//YOn83YJWjDUIRe4GOc4yExT1uWSwb8mGeWS1q7u+1ZY064Veu+R908jiGVQbCjfYsK2154qjsPnqEBYV0nnHdaSG4VWobk+wOd8+sCEwEAAA==

:recycle: This comment has been updated with latest results.
author_association: NONE
user: login: sethrj
created_at: 2025-03-14T14:14:23Z
body: Currently fails until #1665 is merged.
author_association: MEMBER