Repository: eth-cscs/DLA-Future
Pull request #1313: CI on beverin
Author: login: rasolca
Labels: name: Type:New Feature
description: New feature or request
name: Category:CI
name: Priority:High
Created: 2025-04-01T14:46:42Z | Merged: 2025-04-23T11:09:08Z

Modified rocm release pipeline:
- clang -> gcc12 (cray-mpich%clang can't be installed)
- added test stage that run on beverin's MI250x

Open tasks for other PRs:
- stdexec pipeline
- upgrade to rocm 6.3 (update #1242 and #1251?)
- add second gpu architecture and run tests on MI300a too.


Diff:
diff --git a/ci/.gitlab-ci.yml b/ci/.gitlab-ci.yml
index 1ca1abe7e9..3359b5713f 100644
--- a/ci/.gitlab-ci.yml
+++ b/ci/.gitlab-ci.yml
@@ -8,6 +8,10 @@
 # SPDX-License-Identifier: BSD-3-Clause
 #
 include:
+
+  # This tests is needed to quickly check that TDS credentials are valid.
+  - local: 'ci/rocm/test.yml'
+
   - local: 'ci/cpu/asan_ubsan_lsan.yml'
   - local: 'ci/cpu/clang15_release_cxx20.yml'
   - local: 'ci/cpu/clang15_release_stdexec.yml'
@@ -24,5 +28,5 @@ include:
   - local: 'ci/cuda/gcc13_debug.yml'
   - local: 'ci/cuda/gcc13_debug_scalapack.yml'
   - local: 'ci/cuda/gcc13_release_stdexec.yml'
-  - local: 'ci/rocm/clang15_release.yml'
+  - local: 'ci/rocm/gcc12_release.yml'
   - local: 'ci/rocm/clang15_release_stdexec.yml'
diff --git a/ci/base-images/amdgpu-cray-mpich/HOWTO.md b/ci/base-images/amdgpu-cray-mpich/HOWTO.md
new file mode 100644
index 0000000000..a9f7f902a4
--- /dev/null
+++ b/ci/base-images/amdgpu-cray-mpich/HOWTO.md
@@ -0,0 +1,43 @@
+# Modified base image to allow building cray-mpich
+
+## Preparation steps
+
+```
+mkdir lib64
+cp -a /usr/lib64/libxpmem.* lib64/
+
+git clone https://github.com/eth-cscs/alps-cluster-config.git
+
+cp alps-cluster-config/beverin/packages.yaml packages.yaml
+```
+
+## Edit cluster config files
+
+Modify `packages.yaml`:
+```
+   patchelf:
+     require: "@:0.17"
+   libfabric:
+-    buildable: false
+-    externals:
+-    - spec: libfabric@1.15.2.0
+-      prefix: /opt/cray/libfabric/1.15.2.0/
++    require: "@1.15.2"
+   slurm:
+     buildable: false
+     externals:
+```
+Note: The container engine (CE) will replace libfabric with the system one when running the container.
+Make sure to use the same version.
+
+## Build and push
+
+```
+export TAG="v0.2"
+CSCS_REGISTRY="jfrog.svc.cscs.ch/docker-ci-ext/4700071344751697"
+podman login jfrog.svc.cscs.ch
+
+podman build -f build.Dockerfile -t $CSCS_REGISTRY/base-images/rocm_6.0.2-dev-ubuntu-22.04:$TAG
+
+podman push $CSCS_REGISTRY/base-images/rocm_6.0.2-dev-ubuntu-22.04:$TAG
+```
diff --git a/ci/base-images/amdgpu-cray-mpich/build.Dockerfile b/ci/base-images/amdgpu-cray-mpich/build.Dockerfile
new file mode 100644
index 0000000000..ce4064284f
--- /dev/null
+++ b/ci/base-images/amdgpu-cray-mpich/build.Dockerfile
@@ -0,0 +1,5 @@
+FROM docker.io/rocm/dev-ubuntu-22.04:6.0.2
+
+COPY lib64 /usr/lib
+COPY packages.yaml /root/.spack/packages.yaml
+COPY alps-cluster-config/site /root/site
diff --git a/ci/ci-ext-custom.yml b/ci/ci-ext-custom.yml
index edbd757623..4fda4c9f9a 100644
--- a/ci/ci-ext-custom.yml
+++ b/ci/ci-ext-custom.yml
@@ -10,8 +10,30 @@
 include:
   - remote: 'https://gitlab.com/cscs-ci/recipes/-/raw/master/templates/v2/.ci-ext.yml'
 
-# Need custom definition as remote adds SLURM_MPI_TYPE: pmi2
+.dlaf-container-runner-eiger:
+  extends: .container-runner-eiger
+  variables:
+    SLURM_CONSTRAINT: mc
+
 .dlaf-container-runner-daint-gh200:
   extends: .container-runner-daint-gh200
   variables:
+    CSCS_CUDA_MPS: 1
+    # Need to reset as remote adds SLURM_MPI_TYPE: pmi2
     SLURM_MPI_TYPE: cray_shasta
+
+# TDS needs different credentials
+.dlaf-container-runner-beverin:
+  variables:
+    F7T_CLIENT_ID: $F7T_TDS_CLIENT_ID
+    F7T_CLIENT_SECRET: $F7T_TDS_CLIENT_SECRET
+
+.dlaf-container-runner-beverin-mi300:
+  extends:
+    - .container-runner-beverin-mi300
+    - .dlaf-container-runner-beverin
+
+.dlaf-container-runner-beverin-mi200:
+  extends:
+    - .container-runner-beverin-mi200
+    - .dlaf-container-runner-beverin
diff --git a/ci/common-ci.yml b/ci/common-ci.yml
index ee5b27f89a..73433dfba0 100644
--- a/ci/common-ci.yml
+++ b/ci/common-ci.yml
@@ -74,6 +74,11 @@ variables:
   variables:
     COMMON_SPACK_ENVIRONMENT: ci/docker/common.yaml
 
+.build_deps_common_amdgpu:
+  extends: .build_deps_common
+  variables:
+    COMMON_SPACK_ENVIRONMENT: ci/docker/common-amdgpu.yaml
+
 .build_deps_common_gh200:
   timeout: 6 hours
   extends:
@@ -92,7 +97,7 @@ variables:
     - 'echo $DOCKERHUB_TOKEN | podman login docker.io -u $DOCKERHUB_USERNAME --password-stdin'
     - 'echo "INFO: Using NUM_CORES_BUILD_DLAF=$NUM_CORES_BUILD_DLAF"'
   after_script:
-    - podman run -v $PWD/ci/ctest_to_gitlab.sh:/ctest_to_gitlab.sh $DLAF_IMAGE /ctest_to_gitlab.sh "$DLAF_IMAGE" "$USE_CODECOV" "$THREADS_MAX_PER_TASK" "$THREADS_PER_NODE" "$SLURM_CONSTRAINT" "$RUNNER" > pipeline.yml
+    - podman run -v $PWD/ci/ctest_to_gitlab.sh:/ctest_to_gitlab.sh $DLAF_IMAGE /ctest_to_gitlab.sh "$DLAF_IMAGE" "$USE_CODECOV" "$THREADS_MAX_PER_TASK" "$THREADS_PER_NODE" "$RUNNER" > pipeline.yml
   variables:
     PERSIST_IMAGE_NAME: $DLAF_IMAGE
     DOCKER_BUILD_ARGS: '[
@@ -116,7 +121,6 @@ variables:
   variables:
     DLAF_LD_PRELOAD: "/lib/x86_64-linux-gnu/libSegFault.so"
 
-
 .build_common_gh200:
   timeout: 6 hours
   extends:
@@ -128,15 +132,19 @@ variables:
 
 .build_for_eiger:
   variables:
-    RUNNER: ".container-runner-eiger"
-    SLURM_CONSTRAINT: mc
+    RUNNER: ".dlaf-container-runner-eiger"
     THREADS_MAX_PER_TASK: 32
     THREADS_PER_NODE: 256
 
+.build_for_beverin_mi200:
+  variables:
+    RUNNER: ".dlaf-container-runner-beverin-mi200"
+    THREADS_MAX_PER_TASK: 16
+    THREADS_PER_NODE: 256
+
 .build_for_alps_gh200:
   variables:
     RUNNER: ".dlaf-container-runner-daint-gh200"
-    SLURM_CONSTRAINT: gpu
     # 64 / 2 to avoid ranks on multiple sockets for RANK6
     THREADS_MAX_PER_TASK: 32
     THREADS_PER_NODE: 256
diff --git a/ci/ctest_to_gitlab.sh b/ci/ctest_to_gitlab.sh
index d6cd21922b..3b35b91bc8 100755
--- a/ci/ctest_to_gitlab.sh
+++ b/ci/ctest_to_gitlab.sh
@@ -14,8 +14,7 @@ IMAGE="$1"
 USE_CODECOV="$2"
 THREADS_MAX_PER_TASK="$3"
 THREADS_PER_NODE="$4"
-SLURM_CONSTRAINT="$5"
-RUNNER="$6"
+RUNNER="$5"
 
 if [ "$USE_CODECOV" = true ]; then
 STAGES="
@@ -69,8 +68,6 @@ variables:
   FF_TIMESTAMPS: true
   SLURM_EXCLUSIVE: ''
   SLURM_EXACT: ''
-  SLURM_CONSTRAINT: $SLURM_CONSTRAINT
-  CRAY_CUDA_MPS: 1
   MPICH_MAX_THREAD_SAFETY: multiple
 
 {{JOBS}}
diff --git a/ci/docker/codecov.Dockerfile b/ci/docker/codecov.Dockerfile
index ce08e6e77c..3e8950e747 100644
--- a/ci/docker/codecov.Dockerfile
+++ b/ci/docker/codecov.Dockerfile
@@ -47,9 +47,6 @@ RUN cd /usr/local/bin && \
 
 # Make it easy to call our binaries.
 ENV PATH="${BIN}:$PATH"
-ENV NVIDIA_VISIBLE_DEVICES all
-ENV NVIDIA_DRIVER_CAPABILITIES compute,utility
-ENV NVIDIA_REQUIRE_CUDA "cuda>=10.2"
 
 # Used in our ctest wrapper to upload reports
 ENV ENABLE_COVERAGE="YES"
diff --git a/ci/docker/common-amdgpu.yaml b/ci/docker/common-amdgpu.yaml
new file mode 100644
index 0000000000..f5265844bb
--- /dev/null
+++ b/ci/docker/common-amdgpu.yaml
@@ -0,0 +1,52 @@
+#
+# Distributed Linear Algebra with Future (DLAF)
+#
+# Copyright (c) ETH Zurich
+# All rights reserved.
+#
+# Please, refer to the LICENSE file in the root directory.
+# SPDX-License-Identifier: BSD-3-Clause
+#
+
+packages:
+  all:
+    target: [x86_64]
+  # Set intel MKL as default blas, lapack and scalapack provider.
+  # Can be overwritten in environments if needed.
+  blas:
+    require: 'intel-oneapi-mkl'
+  lapack:
+    require: 'intel-oneapi-mkl'
+  scalapack:
+    require: 'intel-oneapi-mkl'
+  mpi:
+    ### Workaround until CE provides full MPI substitution.
+    require: 'cray-mpich'
+  blaspp:
+    variants:
+      - '~cuda'
+      - '~openmp'
+      - '~rocm'
+  intel-oneapi-mkl:
+    require:
+      - 'threads=openmp'
+  openblas:
+    variants:
+      - 'threads=openmp'
+  openssl:
+    # without version spack concretizes openssl@3.3.1 that doesn't build due to deprecation
+    require:
+      - '@3.3.2'
+  libfabric:
+    require:
+      - '@1.15.2'
+  hwloc:
+    variants:
+      - '~libxml2'
+  git:
+    # Force git as non-buildable to allow deprecated versions in environments
+    # https://github.com/spack/spack/pull/30040
+    buildable: false
+  stdexec:
+    require:
+      - '@git.nvhpc-25.03.rc1=25.03'
diff --git a/ci/docker/deploy.Dockerfile b/ci/docker/deploy.Dockerfile
index 6ee2b07363..62871c1dbc 100644
--- a/ci/docker/deploy.Dockerfile
+++ b/ci/docker/deploy.Dockerfile
@@ -36,9 +36,6 @@ RUN mkdir -p ${BIN} && cp -L ${SOURCE}/ci/{mpi-ctest,check-threads} ${BIN}
 
 # Make it easy to call our binaries.
 ENV PATH="${BIN}:$PATH"
-ENV NVIDIA_VISIBLE_DEVICES all
-ENV NVIDIA_DRIVER_CAPABILITIES compute,utility
-ENV NVIDIA_REQUIRE_CUDA "cuda>=10.2"
 
 # Automatically print stacktraces on segfault
 ARG DLAF_LD_PRELOAD
diff --git a/ci/docker/release-rocm602.yaml b/ci/docker/release-rocm602.yaml
index d78715d193..c185a68913 100644
--- a/ci/docker/release-rocm602.yaml
+++ b/ci/docker/release-rocm602.yaml
@@ -10,7 +10,7 @@
 
 spack:
   include:
-  - /spack_environment/common.yaml
+  - /spack_environment/common-amdgpu.yaml
 
   view: false
   concretizer:
@@ -18,7 +18,7 @@ spack:
       true
 
   specs:
-  - dla-future@master +rocm amdgpu_target=gfx90a:xnack- +mpi_gpu_aware +miniapps +ci-test
+  - dla-future@master +rocm amdgpu_target=gfx90a:xnack- +miniapps +ci-test
 
   packages:
     all:
@@ -30,11 +30,9 @@ spack:
       require:: openblas
     lapack:
       require:: openblas
-    mpich:
+    cray-mpich:
       require:
         - '~rocm'
-        - 'device=ch3'
-        - 'netmod=tcp'
     llvm-amdgpu:
       externals:
       - spec: llvm-amdgpu@6.0.2 ~rocm-device-libs
diff --git a/ci/mpi-ctest b/ci/mpi-ctest
index 5d10a6ea9c..d812f97c94 100755
--- a/ci/mpi-ctest
+++ b/ci/mpi-ctest
@@ -35,6 +35,31 @@ then
     export CUDA_VISIBLE_DEVICES=$(( SLURM_LOCALID % 4 ))
     echo "Rank $SLURM_PROCID: set CUDA_VISIBLE_DEVICES=$CUDA_VISIBLE_DEVICES"
 fi
+if [[ $DLAF_CI_BIND_GPU == "ALPS-MI250X" ]]
+then
+    case $SLURM_LOCALID in
+      0)
+        export ROCR_VISIBLE_DEVICES=4;;
+      1)
+        export ROCR_VISIBLE_DEVICES=5;;
+      2)
+        export ROCR_VISIBLE_DEVICES=2;;
+      3)
+        export ROCR_VISIBLE_DEVICES=3;;
+      4)
+        export ROCR_VISIBLE_DEVICES=6;;
+      5)
+        export ROCR_VISIBLE_DEVICES=7;;
+      6)
+        export ROCR_VISIBLE_DEVICES=0;;
+      7)
+        export ROCR_VISIBLE_DEVICES=1;;
+      *)
+        echo "Too many local Ranks!" && exit 1;;
+    esac
+
+    echo "Rank $SLURM_PROCID: set ROCR_VISIBLE_DEVICES=$ROCR_VISIBLE_DEVICES"
+fi
 
 # Run the tests, only output on the first rank
 if [[ $SLURM_PROCID == "0" ]]; then
diff --git a/ci/rocm/clang15_release.yml b/ci/rocm/clang15_release.yml
deleted file mode 100644
index 96dc76e3fe..0000000000
--- a/ci/rocm/clang15_release.yml
+++ /dev/null
@@ -1,30 +0,0 @@
-#
-# Distributed Linear Algebra with Future (DLAF)
-#
-# Copyright (c) ETH Zurich
-# All rights reserved.
-#
-# Please, refer to the LICENSE file in the root directory.
-# SPDX-License-Identifier: BSD-3-Clause
-#
-include:
-  - local: 'ci/common-ci.yml'
-
-rocm clang15 release deps:
-  extends: .build_deps_common
-  variables:
-    BASE_IMAGE: docker.io/rocm/dev-ubuntu-22.04:6.0.2
-    EXTRA_APTGET: "clang-15 libomp-15-dev rocblas rocblas-dev rocsolver rocsolver-dev llvm-amdgpu rocm-device-libs"
-    COMPILER: clang@15
-    USE_ROCBLAS: "ON"
-    SPACK_ENVIRONMENT: ci/docker/release-rocm602.yaml
-    DEPS_IMAGE: $CSCS_REGISTRY_PATH/rocm-clang15-release/deps
-
-rocm clang15 release build:
-  extends:
-    - .build_common
-  after_script: null
-  needs:
-    - rocm clang15 release deps
-  variables:
-    DLAF_IMAGE: $CSCS_REGISTRY_PATH/rocm-clang15-release/dlaf:$CI_COMMIT_SHA
diff --git a/ci/rocm/gcc12_release.yml b/ci/rocm/gcc12_release.yml
new file mode 100644
index 0000000000..cdaf4d71c9
--- /dev/null
+++ b/ci/rocm/gcc12_release.yml
@@ -0,0 +1,44 @@
+#
+# Distributed Linear Algebra with Future (DLAF)
+#
+# Copyright (c) ETH Zurich
+# All rights reserved.
+#
+# Please, refer to the LICENSE file in the root directory.
+# SPDX-License-Identifier: BSD-3-Clause
+#
+include:
+  - local: 'ci/common-ci.yml'
+
+rocm gcc12 release deps:
+  extends: .build_deps_common_amdgpu
+  variables:
+    BASE_IMAGE: $CSCS_REGISTRY_PATH/base-images/rocm_6.0.2-dev-ubuntu-22.04:v0.2
+    EXTRA_APTGET: "g++-12 gcc-12 gfortran-12 rocblas rocblas-dev rocsolver rocsolver-dev llvm-amdgpu rocm-device-libs libjansson4"
+    COMPILER: gcc@12
+    USE_ROCBLAS: "ON"
+    SPACK_ENVIRONMENT: ci/docker/release-rocm602.yaml
+    DEPS_IMAGE: $CSCS_REGISTRY_PATH/rocm-gcc12-release/deps
+
+rocm gcc12 release build:
+  extends:
+    - .build_common
+    - .build_for_beverin_mi200
+  needs:
+    - rocm gcc12 release deps
+  variables:
+    DLAF_IMAGE: $CSCS_REGISTRY_PATH/rocm-gcc12-release/dlaf:$CI_COMMIT_SHA
+
+rocm gcc12 release test:
+  extends: .run_common
+  variables:
+    DLAF_CI_BIND_GPU: "ALPS-MI250X"
+    PIKA_MPI_ENABLE_POOL: 1
+    PIKA_MPI_COMPLETION_MODE: 30
+    MPICH_GPU_SUPPORT_ENABLED: 0
+  needs:
+    - rocm gcc12 release build
+  trigger:
+    include:
+      - artifact: pipeline.yml
+        job: rocm gcc12 release build
diff --git a/ci/rocm/test.yml b/ci/rocm/test.yml
new file mode 100644
index 0000000000..906d0ce241
--- /dev/null
+++ b/ci/rocm/test.yml
@@ -0,0 +1,29 @@
+#
+# Distributed Linear Algebra with Future (DLAF)
+#
+# Copyright (c) ETH Zurich
+# All rights reserved.
+#
+# Please, refer to the LICENSE file in the root directory.
+# SPDX-License-Identifier: BSD-3-Clause
+#
+include:
+  - local: 'ci/common-ci.yml'
+
+test_mi200:
+  image: docker.io/rocm/dev-ubuntu-22.04:6.0.2
+  stage: build_deps
+  extends: .dlaf-container-runner-beverin-mi200
+  variables:
+    FF_TIMESTAMPS: true
+    SLURM_EXCLUSIVE: ''
+    SLURM_EXACT: ''
+    MPICH_MAX_THREAD_SAFETY: multiple
+    SLURM_NTASKS: 1
+    SLURM_TIMELIMIT: '0:30:00'
+    SLURM_UNBUFFEREDIO: 1
+    SLURM_WAIT: 0
+    PULL_IMAGE: 'YES'
+    USE_MPI: 'YES'
+    DISABLE_AFTER_SCRIPT: 'YES'
+  script: rocm-smi
diff --git a/include/dlaf/matrix/copy_tile.h b/include/dlaf/matrix/copy_tile.h
index 8027c04f99..302ca599fd 100644
--- a/include/dlaf/matrix/copy_tile.h
+++ b/include/dlaf/matrix/copy_tile.h
@@ -81,6 +81,9 @@ template <typename T>
 struct CopyTile<T, Device::CPU, Device::GPU> {
   static void call(const matrix::Tile<const T, Device::CPU>& source,
                    const matrix::Tile<T, Device::GPU>& destination) {
+    if (source.size().isEmpty())
+      return;
+
     const std::size_t m = to_sizet(source.size().rows());
     const std::size_t n = to_sizet(source.size().cols());
     const std::size_t ld_source = to_sizet(source.ld());
@@ -92,6 +95,9 @@ struct CopyTile<T, Device::CPU, Device::GPU> {
 
   static void call(const matrix::Tile<const T, Device::CPU>& source,
                    const matrix::Tile<T, Device::GPU>& destination, whip::stream_t stream) {
+    if (source.size().isEmpty())
+      return;
+
     const std::size_t m = to_sizet(source.size().rows());
     const std::size_t n = to_sizet(source.size().cols());
     const std::size_t ld_source = to_sizet(source.ld());
@@ -106,6 +112,9 @@ template <typename T>
 struct CopyTile<T, Device::GPU, Device::CPU> {
   static void call(const matrix::Tile<const T, Device::GPU>& source,
                    const matrix::Tile<T, Device::CPU>& destination) {
+    if (source.size().isEmpty())
+      return;
+
     const std::size_t m = to_sizet(source.size().rows());
     const std::size_t n = to_sizet(source.size().cols());
     const std::size_t ld_source = to_sizet(source.ld());
@@ -117,6 +126,9 @@ struct CopyTile<T, Device::GPU, Device::CPU> {
 
   static void call(const matrix::Tile<const T, Device::GPU>& source,
                    const matrix::Tile<T, Device::CPU>& destination, whip::stream_t stream) {
+    if (source.size().isEmpty())
+      return;
+
     const std::size_t m = to_sizet(source.size().rows());
     const std::size_t n = to_sizet(source.size().cols());
     const std::size_t ld_source = to_sizet(source.ld());
@@ -131,6 +143,9 @@ template <typename T>
 struct CopyTile<T, Device::GPU, Device::GPU> {
   static void call(const matrix::Tile<const T, Device::GPU>& source,
                    const matrix::Tile<T, Device::GPU>& destination) {
+    if (source.size().isEmpty())
+      return;
+
     const std::size_t m = to_sizet(source.size().rows());
     const std::size_t n = to_sizet(source.size().cols());
     const std::size_t ld_source = to_sizet(source.ld());
@@ -142,6 +157,9 @@ struct CopyTile<T, Device::GPU, Device::GPU> {
 
   static void call(const matrix::Tile<const T, Device::GPU>& source,
                    const matrix::Tile<T, Device::GPU>& destination, whip::stream_t stream) {
+    if (source.size().isEmpty())
+      return;
+
     const std::size_t m = to_sizet(source.size().rows());
     const std::size_t n = to_sizet(source.size().cols());
     const std::size_t ld_source = to_sizet(source.ld());


Reviews:
user: login: rasolca
state: COMMENTED
author_association: COLLABORATOR
submitted_at: 2025-04-05T10:43:34Z
user: login: msimberg
state: CHANGES_REQUESTED
author_association: COLLABORATOR
submitted_at: 2025-04-07T07:59:45Z
user: login: rasolca
state: COMMENTED
author_association: COLLABORATOR
submitted_at: 2025-04-22T13:24:22Z
user: login: msimberg
state: APPROVED
author_association: COLLABORATOR
submitted_at: 2025-04-23T11:07:02Z

Review comments:
diff_hunk: @@ -24,5 +28,5 @@ include:
   - local: 'ci/cuda/gcc13_debug.yml'
   - local: 'ci/cuda/gcc13_debug_scalapack.yml'
   - local: 'ci/cuda/gcc13_release_stdexec.yml'
-  - local: 'ci/rocm/clang15_release.yml'
+  - local: 'ci/rocm/gcc12_release.yml'
   - local: 'ci/rocm/clang15_release_stdexec.yml'
path: ci/.gitlab-ci.yml
user: login: rasolca
body: `ci/rocm/clang15_release_stdexec` is still build only. Will be updated in a future PR.
created_at: 2025-04-05T10:43:34Z
author_association: COLLABORATOR
diff_hunk: @@ -8,6 +8,10 @@
 # SPDX-License-Identifier: BSD-3-Clause
 #
 include:
+
+  # This tests is needed to quickly check that TDS credentials are valid.
+  - local: 'ci/rocm/test.yml'
path: ci/.gitlab-ci.yml
user: login: msimberg
body: Is this meant to stay? Or can be removed?
created_at: 2025-04-07T07:52:43Z
author_association: COLLABORATOR
diff_hunk: @@ -10,8 +10,30 @@
 include:
   - remote: 'https://gitlab.com/cscs-ci/recipes/-/raw/master/templates/v2/.ci-ext.yml'
 
-# Need custom definition as remote adds SLURM_MPI_TYPE: pmi2
+.dlaf-container-runner-eiger:
+  extends: .container-runner-eiger
+  variables:
+    SLURM_CONSTRAINT: mc
+
 .dlaf-container-runner-daint-gh200:
   extends: .container-runner-daint-gh200
   variables:
+    CRAY_CUDA_MPS: 1
path: ci/ci-ext-custom.yml
user: login: msimberg
body: ```suggestion
    CSCS_CUDA_MPS: 1
```
`CRAY_CUDA_MPS` still works, but `CSCS_CUDA_MPS` is preferred (https://confluence.cscs.ch/spaces/KB/pages/868813057/Runners#Runners-container-runner).
created_at: 2025-04-07T07:55:48Z
author_association: COLLABORATOR
diff_hunk: @@ -74,6 +74,11 @@ variables:
   variables:
     COMMON_SPACK_ENVIRONMENT: ci/docker/common.yaml
 
+.build_deps_common_mi:
path: ci/common-ci.yml
user: login: msimberg
body: Purely subjective, but would something like this be clearer:
```suggestion
.build_deps_common_amdgpu:
```
?
created_at: 2025-04-07T07:56:54Z
author_association: COLLABORATOR
diff_hunk: @@ -8,6 +8,10 @@
 # SPDX-License-Identifier: BSD-3-Clause
 #
 include:
+
+  # This tests is needed to quickly check that TDS credentials are valid.
+  - local: 'ci/rocm/test.yml'
path: ci/.gitlab-ci.yml
user: login: rasolca
body: Meant to stay. Can immediately spot if TDS credentials has been messed up.
(Already happened)
created_at: 2025-04-22T13:24:22Z
author_association: COLLABORATOR
diff_hunk: @@ -0,0 +1,43 @@
+# Modified base image to allow building cray-mpich
+
+## Preparation steps
+
+```
+mkdir lib64
+cp -a /usr/lib64/libxpmem.* lib64/
+
+git clone https://github.com/eth-cscs/alps-cluster-config.git
+
+cp alps-cluster-config/beverin/packages.yaml packages.yaml
+```
+
+## Edit cluster config files
+
+Modify `packages.yaml`:
+```
+   patchelf:
+     require: "@:0.17"
+   libfabric:
+-    buildable: false
+-    externals:
+-    - spec: libfabric@1.15.2.0
+-      prefix: /opt/cray/libfabric/1.15.2.0/
++    require: "@1.15.2"
path: ci/base-images/amdgpu-cray-mpich/HOWTO.md
user: login: msimberg
body: Should we be building with 1.22.X now that daint has been updated? I think that's the version the container engine will inject nowadays. I also think building with 1.15 will keep working as they are ABI compatible. I also don't think we're affected by the issues that NCCL/aws-ofi-nccl has with different libfabric versions, but not 100% sure.
created_at: 2025-04-23T11:04:37Z
author_association: COLLABORATOR
diff_hunk: @@ -10,15 +10,15 @@
 
 spack:
   include:
-  - /spack_environment/common.yaml
+  - /spack_environment/common-amdgpu.yaml
 
   view: false
   concretizer:
     unify:
       true
 
   specs:
-  - dla-future@master +rocm amdgpu_target=gfx90a:xnack- +mpi_gpu_aware +miniapps +ci-test
path: ci/docker/release-rocm602.yaml
user: login: msimberg
body: `+mpi_gpu_aware` removed because it doesn't work or for some other reason?
created_at: 2025-04-23T11:06:34Z
author_association: COLLABORATOR

PR comments:
user: login: rasolca
created_at: 2025-04-01T14:46:52Z
body: cscs-ci run
author_association: COLLABORATOR
user: login: rasolca
created_at: 2025-04-01T14:51:32Z
body: cscs-ci run
author_association: COLLABORATOR
user: login: rasolca
created_at: 2025-04-01T14:55:11Z
body: cscs-ci run
author_association: COLLABORATOR
user: login: rasolca
created_at: 2025-04-01T14:59:59Z
body: cscs-ci run
author_association: COLLABORATOR
user: login: rasolca
created_at: 2025-04-01T15:02:02Z
body: cscs-ci run
author_association: COLLABORATOR
user: login: rasolca
created_at: 2025-04-01T16:49:47Z
body: cscs-ci run

author_association: COLLABORATOR
user: login: rasolca
created_at: 2025-04-01T19:09:51Z
body: cscs-ci run
author_association: COLLABORATOR
user: login: rasolca
created_at: 2025-04-01T19:21:49Z
body: cscs-ci run
author_association: COLLABORATOR
user: login: rasolca
created_at: 2025-04-01T19:23:47Z
body: cscs-ci run
author_association: COLLABORATOR
user: login: rasolca
created_at: 2025-04-01T19:25:05Z
body: cscs-ci run
author_association: COLLABORATOR
user: login: rasolca
created_at: 2025-04-01T19:30:08Z
body: cscs-ci run
author_association: COLLABORATOR
user: login: albestro
created_at: 2025-04-03T15:10:56Z
body: cscs-ci run
author_association: COLLABORATOR
user: login: albestro
created_at: 2025-04-03T15:12:33Z
body: cscs-ci run
author_association: COLLABORATOR
user: login: rasolca
created_at: 2025-04-03T15:56:50Z
body: cscs-ci run
author_association: COLLABORATOR
user: login: rasolca
created_at: 2025-04-03T16:03:05Z
body: cscs-ci run
author_association: COLLABORATOR
user: login: rasolca
created_at: 2025-04-03T16:06:51Z
body: cscs-ci run
author_association: COLLABORATOR
user: login: rasolca
created_at: 2025-04-03T16:18:48Z
body: cscs-ci run
author_association: COLLABORATOR
user: login: rasolca
created_at: 2025-04-03T16:23:21Z
body: cscs-ci run
author_association: COLLABORATOR
user: login: rasolca
created_at: 2025-04-03T16:37:57Z
body: cscs-ci run
author_association: COLLABORATOR
user: login: rasolca
created_at: 2025-04-03T17:00:25Z
body: cscs-ci run
author_association: COLLABORATOR
user: login: rasolca
created_at: 2025-04-04T08:56:46Z
body: cscs-ci run
author_association: COLLABORATOR
user: login: rasolca
created_at: 2025-04-04T11:51:48Z
body: cscs-ci run
author_association: COLLABORATOR
user: login: rasolca
created_at: 2025-04-04T14:14:19Z
body: cscs-ci run
author_association: COLLABORATOR
user: login: rasolca
created_at: 2025-04-04T15:03:05Z
body: cscs-ci run
author_association: COLLABORATOR
user: login: rasolca
created_at: 2025-04-04T15:59:54Z
body: cscs-ci run
author_association: COLLABORATOR
user: login: rasolca
created_at: 2025-04-05T10:45:35Z
body: cscs-ci run
author_association: COLLABORATOR
user: login: rasolca
created_at: 2025-04-05T11:42:47Z
body: cscs-ci run
author_association: COLLABORATOR
user: login: codecov-commenter
created_at: 2025-04-05T13:35:21Z
body: :warning: Please install the !['codecov app svg image'](https://github.com/codecov/engineering-team/assets/152432831/e90313f4-9d3a-4b63-8b54-cfe14e7ec20d) to ensure uploads and comments are reliably processed by Codecov.

## [Codecov](https://app.codecov.io/gh/eth-cscs/DLA-Future/pull/1313?dropdown=coverage&src=pr&el=h1&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=eth-cscs) Report
All modified and coverable lines are covered by tests :white_check_mark:
> Project coverage is 95.05%. Comparing base [(`33f48c0`)](https://app.codecov.io/gh/eth-cscs/DLA-Future/commit/33f48c0f87bddf0016086cad2fd374313d76f50a?dropdown=coverage&el=desc&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=eth-cscs) to head [(`bb27e9c`)](https://app.codecov.io/gh/eth-cscs/DLA-Future/commit/bb27e9cf11a2bef4f10d5c7e0be83d2f287d803e?dropdown=coverage&el=desc&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=eth-cscs).
> Report is 4 commits behind head on master.

:exclamation: Your organization needs to install the [Codecov GitHub app](https://github.com/apps/codecov/installations/select_target) to enable full functionality.

<details><summary>Additional details and impacted files</summary>


```diff
@@           Coverage Diff           @@
##           master    #1313   +/-   ##
=======================================
  Coverage   95.05%   95.05%           
=======================================
  Files         141      141           
  Lines        8658     8658           
  Branches     1110     1110           
=======================================
  Hits         8230     8230           
  Misses        241      241           
  Partials      187      187           
```

</details>

[:umbrella: View full report in Codecov by Sentry](https://app.codecov.io/gh/eth-cscs/DLA-Future/pull/1313?dropdown=coverage&src=pr&el=continue&utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=eth-cscs).   
:loudspeaker: Have feedback on the report? [Share it here](https://about.codecov.io/codecov-pr-comment-feedback/?utm_medium=referral&utm_source=github&utm_content=comment&utm_campaign=pr+comments&utm_term=eth-cscs).

<details><summary> :rocket: New features to boost your workflow: </summary>

- :snowflake: [Test Analytics](https://docs.codecov.com/docs/test-analytics): Detect flaky tests, report on failures, and find test suite problems.
</details>
author_association: NONE
user: login: rasolca
created_at: 2025-04-22T13:34:51Z
body: cscs-ci run
author_association: COLLABORATOR