Repository: JuliaParallel/MPI.jl
Pull request #838 (2024-06-18 16:55:30)

Document how to use MPItrampoline
Closes https://github.com/JuliaParallel/MPI.jl/issues/837.


Patch:
diff --git a/.buildkite/pipeline.yml b/.buildkite/pipeline.yml
index f386b6f..e890ab8 100644
--- a/.buildkite/pipeline.yml
+++ b/.buildkite/pipeline.yml
@@ -5,7 +5,7 @@
       key: "cuda-build-openmpi"
       agents:
         queue: "juliagpu"
-        cuda: "11.0"
+        cuda: "*"
       env:
         OPENMPI_VER: "4.1"
         OPENMPI_VER_FULL: "4.1.4"
@@ -44,61 +44,26 @@
 
     - wait
 
-    - label: "Tests -- Julia 1.6"
+    - label: "Tests -- Julia {{matrix.version}}"
+      matrix:
+        setup:
+          version:
+            - "1.6"
+            - "1.7"
+            - "1.8"
+            - "1.9"
+            - "1.10"
+      concurrency: 1
+      concurrency_group: mpi_cuda
       plugins:
         - JuliaCI/julia#v1:
-            version: "1.6"
+            version: "{{matrix.version}}"
             persist_depot_dirs: packages,artifacts,compiled
       agents:
         queue: "juliagpu"
-        cuda: "11.0"
+        cuda: "*"
       if: build.message !~ /\[skip tests\]/
-      timeout_in_minutes: 60
-      env:
-        JULIA_MPI_TEST_NPROCS: 2
-        JULIA_MPI_PATH: "${BUILDKITE_BUILD_CHECKOUT_PATH}/openmpi"
-        OMPI_ALLOW_RUN_AS_ROOT: 1
-        OMPI_ALLOW_RUN_AS_ROOT_CONFIRM: 1
-        OMPI_MCA_btl_vader_single_copy_mechanism: 'none' # https://github.com/open-mpi/ompi/issues/4948
-        OPAL_PREFIX: "${BUILDKITE_BUILD_CHECKOUT_PATH}/openmpi" # Should we set this for the user?
-        JULIA_CUDA_MEMORY_POOL: "none"
-      commands: |
-        echo "--- Configure MPI"
-        buildkite-agent artifact download --step "cuda-build-openmpi" mpi-prefix.tar.gz .
-        mkdir -p $${JULIA_MPI_PATH}
-        tar -zxf mpi-prefix.tar.gz --strip-components 1 -C $${JULIA_MPI_PATH}
-        export PATH=$${JULIA_MPI_PATH}/bin:$${PATH}
-        export LD_LIBRARY_PATH=$${JULIA_MPI_PATH}/lib:$${LD_LIBRARY_PATH}
-
-        echo "--- Setup Julia packages"
-        julia --color=yes --project=. -e '
-            import Pkg
-            Pkg.develop(; path = joinpath(pwd(), "lib", "MPIPreferences"))
-            '
-        julia --color=yes --project=test -e '
-            using Pkg
-            Pkg.develop(path="lib/MPIPreferences")
-            using MPIPreferences
-            MPIPreferences.use_system_binary(export_prefs=true)
-            rm("test/Manifest.toml")
-            '
-
-        echo "+++ Run tests"
-        julia --color=yes --project=. -e '
-           import Pkg
-           Pkg.test("MPI"; test_args=["--backend=CUDA"])
-           '
-
-    - label: "Tests -- Julia latest"
-      plugins:
-        - JuliaCI/julia#v1:
-            version: "1"
-            persist_depot_dirs: packages,artifacts,compiled
-      agents:
-        queue: "juliagpu"
-        cuda: "11.0"
-      if: build.message !~ /\[skip tests\]/
-      timeout_in_minutes: 60
+      timeout_in_minutes: 90
       env:
         JULIA_MPI_TEST_NPROCS: 2
         JULIA_MPI_PATH: "${BUILDKITE_BUILD_CHECKOUT_PATH}/openmpi"
@@ -141,34 +106,38 @@
       key: "rocm-build-openmpi"
       agents:
         queue: "juliagpu"
-        rocm: "*" # todo fix ROCM version
+        rocm: "*"
       env:
-        OPENMPI_VER: "4.1"
-        OPENMPI_VER_FULL: "4.1.4"
-        UCX_VER: "1.13-rc1"
+        OPENMPI_VER: "5.0"
+        OPENMPI_VER_FULL: "5.0.3"
+        UCX_VER: "1.17.0"
         CCACHE_DIR: "/root/ccache"
       commands: |
         echo "--- Install packages"
         apt-get install --yes --no-install-recommends curl ccache
         export PATH="/usr/lib/ccache/:$$PATH"
+
         echo "--- Build UCX"
-        curl -L https://github.com/openucx/ucx/releases/download/v1.13.0-rc1/ucx-1.13.0.tar.gz --output ucx.tar.gz
+        curl -L https://github.com/openucx/ucx/releases/download/v$${UCX_VER}/ucx-$${UCX_VER}.tar.gz --output ucx.tar.gz
         tar -zxf ucx.tar.gz
         pushd ucx-*
         ./configure --with-rocm --enable-mt --prefix=$$(realpath ../mpi-prefix)
         make -j
         make install
         popd
+
         echo "--- Build OpenMPI"
         curl -L https://download.open-mpi.org/release/open-mpi/v$${OPENMPI_VER}/openmpi-$${OPENMPI_VER_FULL}.tar.gz --output openmpi.tar.gz
         tar -zxf openmpi.tar.gz
-        pushd openmpi-*
-        ./configure --with-ucx=$$(realpath ../mpi-prefix) --prefix=$$(realpath ../mpi-prefix)
+        pushd openmpi-$${OPENMPI_VER_FULL}
+        ./configure --with-ucx=$$(realpath ../mpi-prefix) --with-rocm --prefix=$$(realpath ../mpi-prefix)
         make -j
         make install
         popd
+
         echo "--- Package prefix"
         tar -zcf mpi-prefix.tar.gz mpi-prefix/
+
         echo "--- ccache stats"
         ccache -s
       artifact_paths:
@@ -176,18 +145,22 @@
 
     - wait
 
-    - label: "Tests -- Julia latest"
+    - label: "Tests -- Julia {{matrix.version}}"
+      matrix:
+        setup:
+          version:
+            - "1.10"
+      concurrency: 1
+      concurrency_group: mpi_rocm
       plugins:
         - JuliaCI/julia#v1:
-            version: "1" # failing on 1.8
+            version: "{{matrix.version}}"
             persist_depot_dirs: packages,artifacts,compiled
       agents:
         queue: "juliagpu"
-        rocm: "*" # todo fix ROCM version
+        rocm: "*"
       if: build.message !~ /\[skip tests\]/
-      timeout_in_minutes: 60
-      soft_fail:
-        - exit_status: 1
+      timeout_in_minutes: 90
       env:
         JULIA_MPI_TEST_NPROCS: 2
         JULIA_MPI_PATH: "${BUILDKITE_BUILD_CHECKOUT_PATH}/openmpi"
@@ -217,6 +190,7 @@
             '
 
         echo "+++ Run tests"
+        export JULIA_MPI_TEST_EXCLUDE="test_allreduce.jl,test_reduce.jl,test_scan.jl"
         julia --color=yes --project=. -e '
            import Pkg
            Pkg.test("MPI"; test_args=["--backend=AMDGPU"])
diff --git a/.github/workflows/CompatHelper.yml b/.github/workflows/CompatHelper.yml
index 1641ea7..5b3053a 100644
--- a/.github/workflows/CompatHelper.yml
+++ b/.github/workflows/CompatHelper.yml
@@ -8,7 +8,7 @@ jobs:
   CompatHelper:
     runs-on: ubuntu-latest
     steps:
-      - uses: julia-actions/setup-julia@latest
+      - uses: julia-actions/setup-julia@v2
         with:
           version: 1.3
       - name: Pkg.add("CompatHelper")
@@ -16,4 +16,4 @@ jobs:
       - name: CompatHelper.main()
         env:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-        run: julia -e 'using CompatHelper; CompatHelper.main()'
+        run: julia -e 'using CompatHelper; CompatHelper.main(; subdirs=["", "test", "docs"])'
diff --git a/.github/workflows/Documenter.yml b/.github/workflows/Documenter.yml
index f5063e5..85fe7a8 100644
--- a/.github/workflows/Documenter.yml
+++ b/.github/workflows/Documenter.yml
@@ -26,10 +26,10 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v4
-      - uses: julia-actions/setup-julia@v1
+      - uses: julia-actions/setup-julia@v2
         with:
           version: '1'
-      - uses: julia-actions/cache@v1
+      - uses: julia-actions/cache@v2
       - name: Install dependencies
         shell: julia --color=yes --project=docs/ {0}
         run: |
diff --git a/.github/workflows/Invalidations.yml b/.github/workflows/Invalidations.yml
index 983364b..e1b0dd2 100644
--- a/.github/workflows/Invalidations.yml
+++ b/.github/workflows/Invalidations.yml
@@ -19,7 +19,7 @@ jobs:
     if: github.base_ref == github.event.repository.default_branch
     runs-on: ubuntu-latest
     steps:
-    - uses: julia-actions/setup-julia@v1
+    - uses: julia-actions/setup-julia@v2
       with:
         version: '1'
     # current branch
diff --git a/Project.toml b/Project.toml
index 7d4ceb4..ae958b5 100644
--- a/Project.toml
+++ b/Project.toml
@@ -1,7 +1,7 @@
 name = "MPI"
 uuid = "da04e1cc-30fd-572f-bb4f-1f8673147195"
 authors = []
-version = "0.20.19"
+version = "0.20.22"
 
 [deps]
 Distributed = "8ba89e20-285c-5b6f-9357-94700520ee1b"
@@ -20,14 +20,14 @@ Sockets = "6462fe0b-24de-5631-8697-dd941f90decc"
 
 [compat]
 Distributed = "1"
-AMDGPU = "0.5.7, 0.6, 0.7, 0.8"
+AMDGPU = "0.6, 0.7, 0.8, 0.9, 1"
 CUDA = "3, 4, 5"
 DocStringExtensions = "0.8, 0.9"
 Libdl = "1"
 MPICH_jll = "4"
 MPIPreferences = "0.1.8"
 MPItrampoline_jll = "5"
-OpenMPI_jll = "4"
+OpenMPI_jll = "4, 5"
 PkgVersion = "0.3"
 PrecompileTools = "1.0.1"
 Requires = "~0.5, 1.0"
diff --git a/README.md b/README.md
index c057b50..c552dcc 100644
--- a/README.md
+++ b/README.md
@@ -7,7 +7,7 @@
 [![codecov.io](https://codecov.io/github/JuliaParallel/MPI.jl/coverage.svg?branch=master)](https://codecov.io/github/JuliaParallel/MPI.jl?branch=master)
 [![Coverage Status](https://coveralls.io/repos/JuliaParallel/MPI.jl/badge.svg?branch=master&service=github)](https://coveralls.io/github/JuliaParallel/MPI.jl?branch=master)
 
-This provides [Julia](http://julialang.org/) interface to the Message Passing Interface ([MPI](http://www.mpi-forum.org/)), roughly inspired by [mpi4py](https://github.com/mpi4py/mpi4py/).
+This provides [Julia](https://julialang.org/) interface to the Message Passing Interface ([MPI](https://www.mpi-forum.org/)), roughly inspired by [mpi4py](https://github.com/mpi4py/mpi4py/).
 
 Please see the [documentation](https://juliaparallel.github.io/MPI.jl/stable/) for instructions on [configuration](https://juliaparallel.github.io/MPI.jl/stable/configuration/) and [usage](https://juliaparallel.github.io/MPI.jl/stable/usage/).
 
diff --git a/docs/Project.toml b/docs/Project.toml
index d456aa0..f0b02b6 100644
--- a/docs/Project.toml
+++ b/docs/Project.toml
@@ -4,4 +4,6 @@ MPI = "da04e1cc-30fd-572f-bb4f-1f8673147195"
 MPIPreferences = "3da0fdf6-3ccc-4f1b-acd9-58baa6c99267"
 
 [compat]
-Documenter = "1"
+Documenter = "1.7"
+MPIPreferences = "0.1"
+MPI = "0.20"
diff --git a/docs/examples/03-reduce.jl b/docs/examples/03-reduce.jl
index 86d22be..cfd3f9c 100644
--- a/docs/examples/03-reduce.jl
+++ b/docs/examples/03-reduce.jl
@@ -31,10 +31,15 @@ function pool(S1::SummaryStat, S2::SummaryStat)
     SummaryStat(m,v,n)
 end
 
+# Register the custom reduction operator.  This is necessary only on platforms
+# where Julia doesn't support closures as cfunctions (e.g. ARM), but can be used
+# on all platforms for consistency.
+MPI.@RegisterOp(pool, SummaryStat)
+
 X = randn(10,3) .* [1,3,7]'
 
 # Perform a scalar reduction
-summ = MPI.Reduce(SummaryStat(X), pool, root, comm)
+summ = MPI.Reduce(SummaryStat(X), pool, comm; root)
 
 if MPI.Comm_rank(comm) == root
     @show summ.var
@@ -42,7 +47,7 @@ end
 
 # Perform a vector reduction:
 # the reduction operator is applied elementwise
-col_summ = MPI.Reduce(mapslices(SummaryStat,X,dims=1), pool, root, comm)
+col_summ = MPI.Reduce(mapslices(SummaryStat,X,dims=1), pool, comm; root)
 
 if MPI.Comm_rank(comm) == root
     col_var = map(summ -> summ.var, col_summ)
diff --git a/docs/examples/05-job_schedule.jl b/docs/examples/05-job_schedule.jl
index 83ce745..0b30871 100644
--- a/docs/examples/05-job_schedule.jl
+++ b/docs/examples/05-job_schedule.jl
@@ -3,8 +3,10 @@
 # assigns one element to each worker to compute the operation.
 # When the worker is finished, the root sends another element
 # until each element is added 100
-# Inspired on
-# https://www.hpc.ntnu.no/ntnu-hpc-group/vilje/user-guide/software/mpi-and-mpi-io-training-tutorial/basic-mpi/job-queue
+# Inspired by https://www.hpc.ntnu.no/vilje/software/mpi-and-mpi-io-training-tutorial/
+# https://www.hpc.ntnu.no/vilje/software/mpi-and-mpi-io-training-tutorial/basic-mpi/job-queue/
+# an updated job_queue.c is available in the basic_mpi/04_job_queue/src subdirectory of
+# the extracted https://www.hpc.ntnu.no/wp-content/uploads/2019/09/mpiexamples.tar.gz
 
 using MPI
 
diff --git a/docs/make.jl b/docs/make.jl
index 2ce24ec..9a1d84f 100644
--- a/docs/make.jl
+++ b/docs/make.jl
@@ -39,7 +39,7 @@ for (example_title, example_md) in EXAMPLES
         println(mdfile, "```")
         println(mdfile, "> mpiexecjl -n 4 julia $example_jl")
         cd(@__DIR__) do
-            write(mdfile, mpiexec(cmd -> read(`$cmd -n 4 $(Base.julia_cmd()) --project $example_jl`)))
+            write(mdfile, read(`$(mpiexec()) -n 4 $(Base.julia_cmd()) --project $example_jl`))
         end
         println(mdfile, "```")
     end
@@ -78,6 +78,10 @@ makedocs(
         ],
         "refindex.md",
     ],
+    # Run linkcheck tests only on CI, to speed up building of documentation locally.  If one
+    # wants to run the linkchecks locally they can set the environment variable `CI=true`.
+    linkcheck = get(ENV, "CI", "false") == "true",
+    linkcheck_useragent = nothing,
 )
 
 deploydocs(
diff --git a/docs/src/configuration.md b/docs/src/configuration.md
index 215605f..0477de7 100644
--- a/docs/src/configuration.md
+++ b/docs/src/configuration.md
@@ -1,14 +1,34 @@
 # Configuration
 
 By default, MPI.jl will download and link against the following MPI implementations:
-- [Microsoft MPI](https://docs.microsoft.com/en-us/message-passing-interface/microsoft-mpi) on Windows
-- [MPICH](http://www.mpich.org/) on all other platforms
+- [Microsoft MPI](https://learn.microsoft.com/en-us/message-passing-interface/microsoft-mpi) on Windows
+- [MPICH](https://www.mpich.org/) on all other platforms
 
 This is suitable for most single-node use cases, but for larger systems, such as HPC
 clusters or multi-GPU machines, you will probably want to configure against a
-system-provided MPI implementation in order to exploit features such as fast network
+specialized MPI implementation in order to exploit features such as fast network
 interfaces and CUDA-aware or ROCm-aware MPI interfaces.
 
+There are three ways to point Julia to an MPI implementation:
+- *jll binaries* (preferred). A jll binary is a package that is built
+  via [BinaryBuilder](https://docs.binarybuilder.org/stable/) and can
+  be downloaded for all architectures. Almost all external libraries
+  (such as CUDA, HDF5, libpng, OpenSSL, ...) are provided as jll
+  binaries because they are quick and easy to download and work out of
+  the box.
+- *system binaries* (for experts only). A system binary is a shared
+  library that you need to configure and build yourself. This is
+  tricky even for experts. The problem is not just build the shared
+  library, in addition one has to be careful to ensure that the shared
+  library is ABI compatible with all other package you are using.
+  (More details on this below.)
+
+If you want to use an MPI implementation on your system, then you can
+either use it as *system binary* (this is for experts only), or you
+can use MPItrampoline as *jll binary*, and point MPItrampoline to the
+other MPI library. This is preferred since it automatically ensures
+ABI compatibility.
+
 The MPIPreferences.jl package allows the user to choose which MPI implementation to use in MPI.jl. It uses
 [Preferences.jl](https://github.com/JuliaPackaging/Preferences.jl) to configure the MPI backend for each
 project separately. This provides a single source of truth that can be used for JLL packages (Julia packages
@@ -23,19 +43,148 @@ julia --project -e 'using Pkg; Pkg.add("MPIPreferences")'
     See [Migration from MPI.jl v0.19 or earlier](@ref) for more information on
     how to migrate your configuration from earlier MPI.jl versions.
 
+
+
+## [Using a jll MPI backend](@id using_jll_mpi)
+
+This is the recommended way to use MPI.jl. By default, MPI.jl will use
+`MPICH_jll` as jll MPI backend.
+
+You can select from four different jll MPI binaries:
+- [`MPICH_jll`](http://www.mpich.org/)),  the default
+- [`OpenMPI_jll`](https://www.open-mpi.org/), an alternative to MPICH
+- [`MPItrampoline_jll`](https://github.com/eschnett/MPItrampoline), a
+  forwarding MPI implementation that uses another MPI implementation
+- [`MicrosoftMPI_jll`](https://docs.microsoft.com/en-us/message-passing-interface/microsoft-mpi)
+  for Windows
+
+For example, to switch to OpenMPI, you would first use MPIPreferenes.jl to switch:
+
+```sh
+julia> using MPIPreferences
+
+julia> MPIPreferences.use_jll_binary("OpenMPI_jll")
+┌ Info: MPIPreferences changed
+└   binary = "OpenMPI_jll"
+```
+
+Next you need to restart Julia (!) and re-instantiate your packages:
+
+```sh
+julia> using Pkg
+
+julia> Pkg.instantiate()
+```
+
+This is necessary because other jll packages (e.g. `HDF5_jll`) may
+depend on MPI, and a different build of `HDF5_jll` needs to be
+installed for ABI compatibility.
+
+To switch back to `MPICH_jll`, repeat the steps above with
+`"MPICH_JLL"` as argument to `MPIPreferences.use_jll_binary`. Don't
+forget to restart Julia and re-instantiate your packages again.
+
+
+
+## [Using MPItrampoline](@id using_mpitrampoline)
+
+MPItrampoline is an easier and safer way to use external MPI
+implementations. MPItrampoline defines an ABI for MPI calls (similar
+to the way
+[`libblastrampoline`](https://github.com/JuliaLinearAlgebra/libblastrampoline)
+works for linear algebra), which allows switching between different
+MPI backends without recompiling code. Unfortunately this ABI is not
+yet standardized (but it might be in [MPI
+5](https://www.mpi-forum.org)!), and therefore you have to install a
+small wrapper library for every MPI implementation that you want to
+use via MPItrampoline.
+
+The general design is:
+- There is a pre-installed MPI implementation on your system (e.g.
+  supporting CUDA or Slingshot)
+- You (or a system administrator) installs
+  [`MPIwrapper`](https://github.com/eschnett/MPIwrapper), which
+  implements the MPI ABI
+
+On the Julia side, the following happens:
+- MPItrampoline provides the regular MPI functionality, internally
+  calling out via the MPI ABI to any MPI implementation
+- Other packages (ADIOS2, HDF5, or MPI.jl) can always rely on
+  MPItrampoline
+- At run time (when starting Julia), you choose which MPI
+  implementation you use by setting an environment variable
+
+
+The documentation for
+[MPItrampoline](https://github.com/eschnett/MPItrampoline) describes
+how to install MPIwrapper. This is possibly as simple as
+
+```sh
+cmake -B build -DMPIEXEC_EXECUTABLE=mpiexec -DCMAKE_BUILD_TYPE=RelWithDebInfo -DCMAKE_INSTALL_PREFIX=$HOME/mpiwrapper
+cmake --build build
+cmake --install build
+```
+
+but nothing is ever simple on an HPC system. It might be necessary to
+load certain modules, or to specify more cmake MPI configuration
+options.
+
+Define an environment variable to select this MPI implemenation:
+
+```sh
+export MPITRAMPOLINE_LIB="$HOME/mpiwrapper/lib/libmpiwrapper.so"
+```
+
+pointing to the shared library provided by the MPIwrapper you just
+built. (If you do not set this environment variable the MPItrampoline
+will fall back onto a built-in MPICH. That is, in principle MPI.jl
+could switch over to always using `MPItrampoline_jll`!)
+
+Next you switch MPI.jl over to using `MPItrampoline_jll` as jll
+binary:
+
+```sh
+julia> using MPIPreferences
+
+julia> MPIPreferences.use_jll_binary("MPItrampoline_jll")
+┌ Info: MPIPreferences changed
+└   binary = "MPItrampoline_jll"
+```
+
+and then restart Julia (see above) and re-instantiate your packages:
+
+```sh
+julia> using Pkg
+
+julia> Pkg.instantiate()
+```
+
+You are ready to test this out:
+```sh
+julia> using MPI
+
+julia> MPI.Get_library_version()
+"MPIwrapper 2.11.0, using MPIABI 2.10.0, wrapping:\nMPICH Version:      4.2.1\nMPICH Release date: Wed Apr 17 15:30:02 CDT 2024\nMPICH ABI:          16:1:4\nMPICH Device:       ch3:nemesis\nMPICH configure:    --build=x86_64-linux-musl --host=x86_64-apple-darwin14 --disable-dependency-tracking --disable-doc --enable-shared=no --enable-static=yes --enable-threads=multiple --with-device=ch3 --prefix=/workspace/destdir/lib/mpich --enable-two-level-namespace\nMPICH CC:           cc -fPIC -DPIC  -fno-common  -O2\nMPICH CXX:          c++ -fPIC -DPIC  -O2\nMPICH F77:          gfortran -fPIC -DPIC  -O2\nMPICH FC:           gfortran -fPIC -DPIC  -O2\nMPICH features:     \n"
+```
+
+
+
 ## [Using a system-provided MPI backend](@id using_system_mpi)
 
+This is an alternative way to using an external MPI implementation
+that does not rely on MPItrampoline.
+
 ### Requirements
 
 MPI.jl requires a shared library installation of a C MPI library, supporting the MPI 3.0
 standard or later. The following MPI implementations should work out-of-the-box with MPI.jl:
 
-- [Open MPI](http://www.open-mpi.org/)
-- [MPICH](http://www.mpich.org/) (v3.1 or later)
-- [Intel MPI](https://software.intel.com/en-us/mpi-library)
-- [Microsoft MPI](https://docs.microsoft.com/en-us/message-passing-interface/microsoft-mpi)
-- [IBM Spectrum MPI](https://www.ibm.com/us-en/marketplace/spectrum-mpi)
-- [MVAPICH](http://mvapich.cse.ohio-state.edu/)
+- [Open MPI](https://www.open-mpi.org/)
+- [MPICH](https://www.mpich.org/) (v3.1 or later)
+- [Intel MPI](https://www.intel.com/content/www/us/en/developer/tools/oneapi/mpi-library.html)
+- [Microsoft MPI](https://learn.microsoft.com/en-us/message-passing-interface/microsoft-mpi)
+- [IBM Spectrum MPI](https://www.ibm.com/products/spectrum-mpi)
+- [MVAPICH](https://mvapich.cse.ohio-state.edu/)
 - [Cray MPICH](https://docs.nersc.gov/development/compilers/wrappers/)
 - [Fujitsu MPI](https://www.fujitsu.com/global/about/resources/publications/technicalreview/2020-03/article07.html#cap-03)
 - [HPE MPT/HMPT](https://support.hpe.com/hpesc/public/docDisplay?docLocale=en_US&docId=a00105727en_us)
@@ -140,7 +289,7 @@ The function of these settings are as follows:
 
 If these are set, the `_format` key will be set to `"1.1"`.
 
-An example of running `MPIPreferences.use_system_library(vendor="cray")` in
+An example of running `MPIPreferences.use_system_binary(vendor="cray")` in
 `PrgEnv-gnu` is:
 
 ```toml
@@ -163,7 +312,7 @@ run on a non-GPU enabled node without needing a separate `LocalPreferences.toml`
 
 The following MPI implementations are provided as JLL packages and automatically obtained when installing MPI.jl:
 
-- `MicrosoftMPI_jll`: [Microsoft MPI](https://docs.microsoft.com/en-us/message-passing-interface/microsoft-mpi) Default for Windows
+- `MicrosoftMPI_jll`: [Microsoft MPI](https://learn.microsoft.com/en-us/message-passing-interface/microsoft-mpi) Default for Windows
 - `MPICH_jll`: [MPICH](https://www.mpich.org/). Default for all other systems
 - `OpenMPI_jll`: [Open MPI](https://www.open-mpi.org/)
 - `MPItrampoline_jll`: [MPItrampoline](https://github.com/eschnett/MPItrampoline): an MPI forwarding layer.
@@ -175,6 +324,8 @@ julia --project -e 'using MPIPreferences; MPIPreferences.use_jll_binary("MPItram
 If you omit the JLL binary name, the default is selected for the respective
 operating system.
 
+
+
 ## Configuration of the MPI.jl testsuite
 
 ### Testing against a different MPI implementation
@@ -216,6 +367,7 @@ The test suite can also be modified by the following variables:
 - `JULIA_MPI_TEST_ABI`: Check that the specified MPI ABI is used for the tests
 
 
+
 ## Migration from MPI.jl v0.19 or earlier
 
 For MPI.jl v0.20, environment variables were used to configure which MPI
diff --git a/docs/src/external.md b/docs/src/external.md
index 9de8ef2..37166fd 100644
--- a/docs/src/external.md
+++ b/docs/src/external.md
@@ -66,5 +66,5 @@ There are two typical solutions to this problem:
 When working with non-Julia libraries or tools, `MPI_Init` may be invoked in another part
 of the execution flow and not via MPI.jl's [`MPI.Init`](@ref) function. This leaves some
 package-internal settings uninitialized. In this case, you need to call
-[`MPI.run_init_hooks()`)(@ref) manually to fully initialize MPI.jl. You may also want to
-consider calling [`MPI.set_default_error_handler_return()`](@ref).
\ No newline at end of file
+[`MPI.run_init_hooks()`](@ref) manually to fully initialize MPI.jl. You may also want to
+consider calling [`MPI.set_default_error_handler_return()`](@ref).
diff --git a/docs/src/index.md b/docs/src/index.md
index e195a3c..c8b992b 100644
--- a/docs/src/index.md
+++ b/docs/src/index.md
@@ -1,8 +1,8 @@
 # MPI.jl
 
-This is a basic [Julia](http://julialang.org/) wrapper for the portable message passing
-system Message Passing Interface ([MPI](http://www.mpi-forum.org/)). Inspiration is taken from
-[mpi4py](http://mpi4py.scipy.org), although we generally follow the C and not the C++ MPI API.
+This is a basic [Julia](https://julialang.org/) wrapper for the portable message passing
+system Message Passing Interface ([MPI](https://www.mpi-forum.org/)). Inspiration is taken from
+[mpi4py](https://mpi4py.readthedocs.io/en/stable/), although we generally follow the C and not the C++ MPI API.
 (The C++ MPI API is deprecated.)
 
 If you use MPI.jl in your work, please cite the following paper:
diff --git a/docs/src/knownissues.md b/docs/src/knownissues.md
index d3e6a2a..660651d 100644
--- a/docs/src/knownissues.md
+++ b/docs/src/knownissues.md
@@ -109,7 +109,7 @@ For further information see
 
 ## UCX
 
-[UCX](https://www.openucx.org/) is a communication framework used by several MPI implementations.
+[UCX](https://openucx.org/) is a communication framework used by several MPI implementations.
 
 ### Memory cache
 
@@ -180,7 +180,7 @@ Make sure to:
   ```
 - Then in Julia, upon loading MPI and CUDA modules, you can check
   - CUDA version: `CUDA.versioninfo()`
-  - If MPI has CUDA: `MPI.has_cuda()`
+  - If MPI has CUDA: [`MPI.has_cuda()`](@ref)
   - If you are using correct MPI library: `MPI.libmpi`
 
 After that, it may be preferred to run the Julia MPI script (as suggested [here](https://discourse.julialang.org/t/cuda-aware-mpi-works-on-system-but-not-for-julia/75060/11)) launching it from a shell script (as suggested [here](https://discourse.julialang.org/t/cuda-aware-mpi-works-on-system-but-not-for-julia/75060/4)).
@@ -195,8 +195,9 @@ Make sure to:
   ```
   julia --project -e 'using Pkg; Pkg.add(["AMDGPU", "MPIPreferences", "MPI"]); using MPIPreferences; MPIPreferences.use_system_binary()'
   ```
-- Then in Julia, upon loading MPI and CUDA modules, you can check
+- Then in Julia, upon loading MPI and AMDGPU modules, you can check
   - AMDGPU version: `AMDGPU.versioninfo()`
+  - If MPI has ROCm: [`MPI.has_rocm()`](@ref)
   - If you are using correct MPI implementation: `MPI.identify_implementation()`
 
 After that, [this script](https://gist.github.com/luraess/c228ec08629737888a18c6a1e397643c) can be used to verify if ROCm-aware MPI is functional (modified after the CUDA-aware version from [here](https://discourse.julialang.org/t/cuda-aware-mpi-works-on-system-but-not-for-julia/75060/11)). It may be preferred to run the Julia ROCm-aware MPI script launching it from a shell script (as suggested [here](https://discourse.julialang.org/t/cuda-aware-mpi-works-on-system-but-not-for-julia/75060/4)).
@@ -209,3 +210,5 @@ However they have two limitations:
 
 * [Julia's C-compatible function pointers](https://docs.julialang.org/en/v1/manual/calling-c-and-fortran-code/index.html#Creating-C-Compatible-Julia-Function-Pointers-1) cannot be used where the `stdcall` calling convention is expected, which is the case for 32-bit Microsoft MPI,
 * closure cfunctions in Julia are based on LLVM trampolines, which are not supported on ARM architecture.
+
+As an alternative [`MPI.@RegisterOp`](@ref) may be used to statically register reduction operations.
diff --git a/docs/src/reference/advanced.md b/docs/src/reference/advanced.md
index 6440fd5..7d4ed0a 100644
--- a/docs/src/reference/advanced.md
+++ b/docs/src/reference/advanced.md
@@ -26,6 +26,7 @@ MPI.Types.duplicate
 
 ```@docs
 MPI.Op
+MPI.@RegisterOp
 ```
 
 ## Info objects
diff --git a/docs/src/reference/library.md b/docs/src/reference/library.md
index 94e3942..302015f 100644
--- a/docs/src/reference/library.md
+++ b/docs/src/reference/library.md
@@ -14,5 +14,7 @@ MPI.MPI_LIBRARY_VERSION_STRING
 ```@docs
 MPI.versioninfo
 MPI.has_cuda
+MPI.has_rocm
+MPI.has_gpu
 MPI.identify_implementation
 ```
diff --git a/docs/src/usage.md b/docs/src/usage.md
index 2b8ebe4..c57eae1 100644
--- a/docs/src/usage.md
+++ b/docs/src/usage.md
@@ -98,7 +98,8 @@ should confirm your MPI implementation to have the ROCm support (AMDGPU) enabled
 [alltoall\_test\_rocm\_multigpu.jl](https://gist.github.com/luraess/a47931d7fb668bd4348a2c730d5489f4) should confirm 
 your ROCm-aware MPI implementation to use multiple AMD GPUs (one GPU per rank).
 
-The status of ROCm (AMDGPU) support cannot currently be queried.
+If using OpenMPI, the status of ROCm support can be checked via the
+[`MPI.has_rocm()`](@ref) function.
 
 ## Writing MPI tests
 
diff --git a/lib/MPIPreferences/src/MPIPreferences.jl b/lib/MPIPreferences/src/MPIPreferences.jl
index e12e6ff..a65623c 100644
--- a/lib/MPIPreferences/src/MPIPreferences.jl
+++ b/lib/MPIPreferences/src/MPIPreferences.jl
@@ -66,7 +66,7 @@ Switches the underlying MPI implementation to one provided by JLL packages. A
 restart of Julia is required for the changes to take effect.
 
 Available options are:
-- `"MicrosoftMPI_jll"` (Only option and default on Winddows)
+- `"MicrosoftMPI_jll"` (Only option and default on Windows)
 - `"MPICH_jll"` (Default on all other platform)
 - `"OpenMPI_jll"`
 - `"MPItrampoline_jll"`
diff --git a/lib/MPIPreferences/src/system.jl b/lib/MPIPreferences/src/system.jl
index 3d16043..f54c730 100644
--- a/lib/MPIPreferences/src/system.jl
+++ b/lib/MPIPreferences/src/system.jl
@@ -8,7 +8,8 @@ module System
     const preloads_env_switch = @load_preference("preloads_env_switch")
     const mpiexec_path = @load_preference("mpiexec")
     mpiexec(;adjust_PATH=true, adjust_LIBPATH=true) = `$mpiexec_path`
-    mpiexec(f;adjust_PATH=true, adjust_LIBPATH=true) = f(`$mpiexec_path`)
+    # The following method may be removed in future releases.
+    Base.@deprecate mpiexec(f;adjust_PATH=true, adjust_LIBPATH=true) f(mpiexec())
 
     libmpi_handle = C_NULL
     function __init__()
diff --git a/src/MPI.jl b/src/MPI.jl
index 7d74e49..5304124 100644
--- a/src/MPI.jl
+++ b/src/MPI.jl
@@ -19,9 +19,27 @@ end
 
 
 function _doc_external(fname)
-"""
-- `$fname` man page: [OpenMPI](https://www.open-mpi.org/doc/current/man3/$fname.3.php), [MPICH](https://www.mpich.org/static/docs/latest/www3/$fname.html)
-"""
+    # Ideally we'd like to use the "latest" version of the docs, but MPICH v4.0
+    # at the moment seems to be the last version with all the docstrings.
+    mpich = "[MPICH](https://www.mpich.org/static/docs/v4.0/www3/$(fname).html)"
+    # All the *_c functions are undocumented in OpenMPI website
+    if !endswith(string(fname), "_c")
+        openmpi = "[OpenMPI](https://docs.open-mpi.org/en/main/man-openmpi/man3/$(fname).3.html)"
+        # Few docstrings aren't available for MPICH
+        if fname in (:MPI_Status_c2f, :MPI_Status_f2c) || startswith(string(fname), "MPI_Type_create_f90_")
+            return """
+            - `$fname` man page: $(openmpi)
+            """
+        else
+            return """
+            - `$fname` man page: $(openmpi), $(mpich)
+            """
+        end
+    else
+        return """
+        - `$fname` man page: $(mpich)
+        """
+    end
 end
 
 """
@@ -112,6 +130,22 @@ function __init__()
         """ ENV["JULIA_MPI_BINARY"]=mpi_env_binary MPIPreferences.binary
     end
 
+    # Set environment variables before dlopen.
+
+    # disable UCX memory cache, since it doesn't work correctly
+    # https://github.com/openucx/ucx/issues/5061
+    if !haskey(ENV, "UCX_MEMTYPE_CACHE")
+        ENV["UCX_MEMTYPE_CACHE"] = "no"
+    end
+
+    # Julia multithreading uses SIGSEGV to sync threads
+    # https://docs.julialang.org/en/v1/devdocs/debuggingtips/#Dealing-with-signals-1
+    # By default, UCX will error if this occurs (issue #337)
+    if !haskey(ENV, "UCX_ERROR_SIGNALS")
+        # default is "SIGILL,SIGSEGV,SIGBUS,SIGFPE"
+        ENV["UCX_ERROR_SIGNALS"] = "SIGILL,SIGBUS,SIGFPE"
+    end
+
     # preload any dependencies of libmpi (if needed, eg. GTL on cray) before
     # dlopen'ing the MPI library: https://github.com/JuliaParallel/MPI.jl/pull/716
     MPIPreferences.dlopen_preloads()
@@ -133,20 +167,6 @@ function __init__()
     # calls don't trigger early `dlopen`-ing of the library.
     Base.invokelatest(API.init_consts)
 
-    # disable UCX memory cache, since it doesn't work correctly
-    # https://github.com/openucx/ucx/issues/5061
-    if !haskey(ENV, "UCX_MEMTYPE_CACHE")
-        ENV["UCX_MEMTYPE_CACHE"] = "no"
-    end
-
-    # Julia multithreading uses SIGSEGV to sync threads
-    # https://docs.julialang.org/en/v1/devdocs/debuggingtips/#Dealing-with-signals-1
-    # By default, UCX will error if this occurs (issue #337)
-    if !haskey(ENV, "UCX_ERROR_SIGNALS")
-        # default is "SIGILL,SIGSEGV,SIGBUS,SIGFPE"
-        ENV["UCX_ERROR_SIGNALS"] = "SIGILL,SIGBUS,SIGFPE"
-    end
-
     if MPIPreferences.binary == "MPItrampoline_jll" && !haskey(ENV, "MPITRAMPOLINE_MPIEXEC")
         ENV["MPITRAMPOLINE_MPIEXEC"] = API.MPItrampoline_jll.mpich_mpiexec_path
     end
diff --git a/src/collective.jl b/src/collective.jl
index a8b7b40..9366fbf 100644
--- a/src/collective.jl
+++ b/src/collective.jl
@@ -782,7 +782,7 @@ Allreduce(obj::T, op, comm::Comm) where {T} =
     Scan!(sendbuf, recvbuf, op, comm::Comm)
     Scan!(sendrecvbuf, op, comm::Comm)
 
-Inclusive prefix reduction (analagous to `accumulate` in Julia): `recvbuf` on rank `i`
+Inclusive prefix reduction (analogous to `accumulate` in Julia): `recvbuf` on rank `i`
 will contain the result of reducing `sendbuf` by `op` from ranks `0:i`.
 
 If only a single buffer `sendrecvbuf` is provided, then operations will be performed
@@ -813,7 +813,7 @@ Scan!(buf, op, comm::Comm) =
 """
     recvbuf = Scan(sendbuf, op, comm::Comm)
 
-Inclusive prefix reduction (analagous to `accumulate` in Julia): `recvbuf` on rank `i`
+Inclusive prefix reduction (analogous to `accumulate` in Julia): `recvbuf` on rank `i`
 will contain the result of reducing `sendbuf` by `op` from ranks `0:i`.
 
 `sendbuf` can also be a scalar, in which case `recvbuf` will also be a scalar of the same
@@ -837,7 +837,7 @@ Scan(object::T, op, comm::Comm) where {T} = Scan!(Ref(object), Ref{T}(), op, com
     Exscan!(sendbuf, recvbuf, op, comm::Comm)
     Exscan!(sendrecvbuf, op, comm::Comm)
 
-Exclusive prefix reduction (analagous to `accumulate` in Julia): `recvbuf` on rank `i`
+Exclusive prefix reduction (analogous to `accumulate` in Julia): `recvbuf` on rank `i`
 will contain the result of reducing `sendbuf` by `op` from ranks `0:i-1`. The
 `recvbuf` on rank `0` is ignored, and the `recvbuf` on rank `1` will contain the contents
 of `sendbuf` on rank `0`.
@@ -869,7 +869,7 @@ Exscan!(buf, op, comm::Comm) = Exscan!(IN_PLACE, buf, op, comm)
 """
     recvbuf = Exscan(sendbuf, op, comm::Comm)
 
-Exclusive prefix reduction (analagous to `accumulate` in Julia): `recvbuf` on rank `i`
+Exclusive prefix reduction (analogous to `accumulate` in Julia): `recvbuf` on rank `i`
 will contain the result of reducing `sendbuf` by `op` from ranks `0:i-1`. The
 `recvbuf` on rank `0` is undefined, and the `recvbuf` on rank `1` will contain the contents
 of `sendbuf` on rank `0`.
diff --git a/src/datatypes.jl b/src/datatypes.jl
index 9dd721f..cefe5aa 100644
--- a/src/datatypes.jl
+++ b/src/datatypes.jl
@@ -149,7 +149,7 @@ end
 
 function Datatype(::Type{T}) where {T}
     global created_datatypes
-    get!(created_datatypes, T) do
+    datatype = get!(created_datatypes, T) do
         datatype = Datatype()
         # lazily initialize so that it can be safely precompiled
         function init()
@@ -162,6 +162,15 @@ function Datatype(::Type{T}) where {T}
         init()
         datatype
     end
+
+    # Make sure the "aligned" size of the type matches the MPI "extent".
+    sz = sizeof(T)
+    al = Base.datatype_alignment(T)
+    mpi_extent = Types.extent(datatype)
+    aligned_size = (0, cld(sz,al)*al)
+    @assert mpi_extent == aligned_size "The MPI extent of type $(T) ($(mpi_extent[2])) does not match the size expected by Julia ($(aligned_size[2]))"
+
+    return datatype
 end
 
 function Base.show(io::IO, datatype::Datatype)
@@ -437,8 +446,10 @@ function create!(newtype::Datatype, ::Type{T}) where {T}
     types = Datatype[]
 
     if isprimitivetype(T)
-        # primitive type
-        szrem = sz = sizeof(T)
+        # This is a primitive type.  Create a type which has size an integer multiple of its
+        # alignment on the Julia side: <https://github.com/JuliaParallel/MPI.jl/issues/853>.
+        al = Base.datatype_alignment(T)
+        szrem = sz = cld(sizeof(T), al) * al
         disp = 0
         for (i,basetype) in (8 => Datatype(UInt64), 4 => Datatype(UInt32), 2 => Datatype(UInt16), 1 => Datatype(UInt8))
             if sz == i
diff --git a/src/environment.jl b/src/environment.jl
index b13e673..7050217 100644
--- a/src/environment.jl
+++ b/src/environment.jl
@@ -1,12 +1,12 @@
 """
-    mpiexec(fn)
+    mpiexec()
 
-A wrapper function for the MPI launcher executable. Calls `fn(cmd)`, where `cmd` is a `Cmd` object of the MPI launcher.
+A wrapper function for the MPI launcher executable. Returns a `Cmd` object pointing to the MPI launcher.
 
 # Usage
 
 ```jldoctest
-julia> mpiexec(cmd -> run(`\$cmd -n 3 echo hello world`));
+julia> run(`\$(mpiexec()) -n 3 echo hello world`);
 hello world
 hello world
 hello world
@@ -287,7 +287,7 @@ It is unaffected by [`MPI.Finalize`](@ref), and is one of the few functions that
 called before [`MPI.Init`](@ref).
 
 # External links
-$(_doc_external("MPI_Intialized"))
+$(_doc_external("MPI_Initialized"))
 """
 function Initialized()
     flag = Ref{Cint}()
@@ -320,10 +320,15 @@ Wtime() = API.MPI_Wtime()
 
 Check if the MPI implementation is known to have CUDA support. Currently only Open MPI
 provides a mechanism to check, so it will return `false` with other implementations
-(unless overriden).
+(unless overridden). For "IBMSpectrumMPI" it will return `true`.
 
-This can be overriden by setting the `JULIA_MPI_HAS_CUDA` environment variable to `true`
+This can be overridden by setting the `JULIA_MPI_HAS_CUDA` environment variable to `true`
 or `false`.
+
+!!! note
+    For OpenMPI or OpenMPI-based implementations you first need to call [Init()](@ref).
+
+See also [`MPI.has_rocm`](@ref) for ROCm support.
 """
 function has_cuda()
     flag = get(ENV, "JULIA_MPI_HAS_CUDA", nothing)
@@ -331,7 +336,7 @@ function has_cuda()
         # Only Open MPI provides a function to check CUDA support
         @static if MPI_LIBRARY == "OpenMPI"
             # int MPIX_Query_cuda_support(void)
-            return 0 != ccall((:MPIX_Query_cuda_support, libmpi), Cint, ())
+            return @ccall libmpi.MPIX_Query_cuda_support()::Bool
         elseif MPI_LIBRARY == "IBMSpectrumMPI"
             return true
         else
@@ -341,3 +346,44 @@ function has_cuda()
         return parse(Bool, flag)
     end
 end
+
+"""
+    MPI.has_rocm()
+
+Check if the MPI implementation is known to have ROCm support. Currently only Open MPI
+provides a mechanism to check, so it will return `false` with other implementations
+(unless overridden).
+
+This can be overridden by setting the `JULIA_MPI_HAS_ROCM` environment variable to `true`
+or `false`.
+
+See also [`MPI.has_cuda`](@ref) for CUDA support.
+"""
+function has_rocm()
+    flag = get(ENV, "JULIA_MPI_HAS_ROCM", nothing)
+    if flag === nothing
+        # Only Open MPI provides a function to check ROCm support
+        @static if MPI_LIBRARY == "OpenMPI" && MPI_LIBRARY_VERSION ≥ v"5"
+            # int MPIX_Query_rocm_support(void)
+            return @ccall libmpi.MPIX_Query_rocm_support()::Bool
+        else
+            return false
+        end
+    else
+        return parse(Bool, flag)
+    end
+end
+
+"""
+    MPI.has_gpu()
+
+Checks if the MPI implementation is known to have GPU support. Currently this checks for the
+following GPUs:
+
+1. CUDA: via [`MPI.has_cuda`](@ref)
+2. ROCm: via [`MPI.has_rocm`](@ref)
+
+See also [`MPI.has_cuda`](@ref) and [`MPI.has_rocm`](@ref) for more fine-grained
+checks.
+"""
+has_gpu() = has_cuda() || has_rocm()
diff --git a/src/operators.jl b/src/operators.jl
index 5ded20f..9f3fed7 100644
--- a/src/operators.jl
+++ b/src/operators.jl
@@ -16,6 +16,7 @@ associative, and if `iscommutative` is true, assumed to be commutative as well.
 - [`Allreduce!`](@ref)/[`Allreduce`](@ref)
 - [`Scan!`](@ref)/[`Scan`](@ref)
 - [`Exscan!`](@ref)/[`Exscan`](@ref)
+- [`@RegisterOp`](@ref)
 """
 mutable struct Op
     val::MPI_Op
@@ -81,21 +82,36 @@ end
 
 function (w::OpWrapper{F,T})(_a::Ptr{Cvoid}, _b::Ptr{Cvoid}, _len::Ptr{Cint}, t::Ptr{MPI_Datatype}) where {F,T}
     len = unsafe_load(_len)
-    @assert isconcretetype(T)
-    a = Ptr{T}(_a)
-    b = Ptr{T}(_b)
-    for i = 1:len
-        unsafe_store!(b, w.f(unsafe_load(a,i), unsafe_load(b,i)), i)
+    if !isconcretetype(T)
+        concrete_T = to_type(Datatype(unsafe_load(t))) # Ptr might actually point to a Julia object so we could unsafe_pointer_to_objref?
+    else
+        concrete_T = T
     end
+    function copy(::Type{T}) where T
+        @assert isconcretetype(T)
+        a = Ptr{T}(_a)
+        b = Ptr{T}(_b)
+        for i = 1:len
+            unsafe_store!(b, w.f(unsafe_load(a,i), unsafe_load(b,i)), i)
+        end
+    end
+    copy(concrete_T)
     return nothing
 end
 
-
 function Op(f, T=Any; iscommutative=false)
     @static if MPI_LIBRARY == "MicrosoftMPI" && Sys.WORD_SIZE == 32
-        error("User-defined reduction operators are not supported on 32-bit Windows.\nSee https://github.com/JuliaParallel/MPI.jl/issues/246 for more details.")
+        error("""
+            User-defined reduction operators are not supported on 32-bit Windows.
+            See https://github.com/JuliaParallel/MPI.jl/issues/246 for more details.
+        """)
     elseif Sys.ARCH ∈ (:aarch64, :ppc64le, :powerpc64le) || startswith(lowercase(String(Sys.ARCH)), "arm")
-        error("User-defined reduction operators are currently not supported on non-Intel architectures.\nSee https://github.com/JuliaParallel/MPI.jl/issues/404 for more details.")
+        error("""
+            User-defined reduction operators are currently not supported on non-Intel architectures.
+            See https://github.com/JuliaParallel/MPI.jl/issues/404 for more details.
+
+            You may want to use `@RegisterOp` to statically register `f`.
+            """)
     end
     w = OpWrapper{typeof(f),T}(f)
     fptr = @cfunction($w, Cvoid, (Ptr{Cvoid}, Ptr{Cvoid}, Ptr{Cint}, Ptr{MPI_Datatype}))
@@ -107,3 +123,70 @@ function Op(f, T=Any; iscommutative=false)
     finalizer(free, op)
     return op
 end
+
+"""
+    @RegisterOp(f, T)
+
+Register a custom operator [`Op`](@ref) using the function `f` statically.
+On platfroms like AArch64, Julia does not support runtime closures,
+being passed to C. The generic version of [`Op`](@ref) uses runtime closures
+to support arbitrary functions being passed as MPI reduction operators.
+`@RegisterOp` statically adds a function to the set of functions allowed as
+as an MPI operator.
+
+```julia
+function my_reduce(x, y)
+    2x+y-x
+end
+MPI.@RegisterOp(my_reduce, Int)
+# ...
+MPI.Reduce!(send_arr, recv_arr, my_reduce, MPI.COMM_WORLD; root=root)
+#...
+```
+!!! warning
+    Note that `@RegisterOp` works be introducing a new method of the generic function `Op`.
+    It can only be used as a top-level statement and may trigger method invalidations.
+
+!!! note
+    `T` can be `Any`, but this will lead to a runtime dispatch.
+"""
+macro RegisterOp(f, T)
+    name_wrapper = gensym(Symbol(f, :_, T, :_wrapper))
+    name_fptr = gensym(Symbol(f, :_, T, :_ptr))
+    name_module = gensym(Symbol(f, :_, T, :_module))
+    # The gist is that we can use a method very similar to how we handle `min`/`max`
+    # but since this might be used from user code we can't use add_load_time_hook!
+    # this is why we introduce a new module that has a `__init__` function.
+    # If this module approach is too costly for loading MPI.jl for internal use we could use
+    # `add_load_time_hook`
+    expr = quote
+        module $(name_module)
+            # import ..$f, ..$T
+            $(Expr(:import, Expr(:., :., :., f), Expr(:., :., :., T))) # Julia 1.6 strugles with import ..$f, ..$T
+            const $(name_wrapper) = $OpWrapper{typeof($f),$T}($f)
+            const $(name_fptr) = Ref(@cfunction($(name_wrapper), Cvoid, (Ptr{Cvoid}, Ptr{Cvoid}, Ptr{Cint}, Ptr{$MPI_Datatype})))
+            function __init__()
+                $(name_fptr)[] = @cfunction($(name_wrapper), Cvoid, (Ptr{Cvoid}, Ptr{Cvoid}, Ptr{Cint}, Ptr{$MPI_Datatype}))
+            end
+            import MPI: Op
+            # we can't create a const Op since MPI needs to be initialized?
+            function Op(::typeof($f), ::Type{<:$T}; iscommutative=false)
+                op = Op($OP_NULL.val, $(name_fptr)[])
+                # int MPI_Op_create(MPI_User_function* user_fn, int commute, MPI_Op* op)
+                $API.MPI_Op_create($(name_fptr)[], iscommutative, op)
+
+                finalizer($free, op)
+            end
+        end
+    end
+    expr.head = :toplevel
+    esc(expr)
+end
+
+@RegisterOp(min, Any)
+@RegisterOp(max, Any)
+@RegisterOp(+, Any)
+@RegisterOp(*, Any)
+@RegisterOp(&, Any)
+@RegisterOp(|, Any)
+@RegisterOp(⊻, Any)
diff --git a/src/pointtopoint.jl b/src/pointtopoint.jl
index b41a2c7..36a2187 100644
--- a/src/pointtopoint.jl
+++ b/src/pointtopoint.jl
@@ -346,7 +346,7 @@ Matching non-blocking probe. Similar to [`MPI.Iprobe`](@ref), except that it
 also returns `msg`, an [`MPI.Message`](@ref) object. 
 
 Checks if there is a message that can be received matching `source`, `tag` and
-`comm`. If so, returns `ismsg = true`, and a [`Message`](@ref) objec `msg`,
+`comm`. If so, returns `ismsg = true`, and a [`Message`](@ref) object `msg`,
 which must be received by either [`MPI.Mrecv!`](@ref) or [`MPI.Imrecv!`](@ref).
 Otherwise `msg` is set to be a null `Message`.
     
@@ -380,7 +380,7 @@ Matching blocking probe. Similar to [`MPI.Probe`](@ref), except that it also
 returns `msg`, an [`MPI.Message`](@ref) object. 
 
 Blocks until a message that can be received matching `source`, `tag` and `comm`,
-returning a [`Message`](@ref) objec `msg`, which must be received by either
+returning a [`Message`](@ref) object `msg`, which must be received by either
 [`MPI.Mrecv!`](@ref) or [`MPI.Imrecv!`](@ref).
     
 The `Status` argument additionally returns the [`Status`](@ref) of the completed


Test patch:
diff --git a/.github/workflows/UnitTests.yml b/.github/workflows/UnitTests.yml
index c8b5000..ec3ecf6 100644
--- a/.github/workflows/UnitTests.yml
+++ b/.github/workflows/UnitTests.yml
@@ -57,11 +57,11 @@ jobs:
     - name: Checkout
       uses: actions/checkout@v4
 
-    - uses: julia-actions/setup-julia@v1
+    - uses: julia-actions/setup-julia@v2
       with:
         arch: ${{ matrix.julia_arch }}
         version: ${{ matrix.julia_version }}
-    - uses: julia-actions/cache@v1
+    - uses: julia-actions/cache@v2
 
     - name: add MPIPreferences
       shell: julia --color=yes --project=. {0}
@@ -71,8 +71,9 @@ jobs:
 
     - uses: julia-actions/julia-runtest@v1
     - uses: julia-actions/julia-processcoverage@v1
-    - uses: codecov/codecov-action@v3
+    - uses: codecov/codecov-action@v5
       with:
+        token: ${{ secrets.CODECOV_TOKEN }}
         files: lcov.info
     - uses: julia-actions/julia-uploadcoveralls@v1
       continue-on-error: true
@@ -90,10 +91,7 @@ jobs:
           - "1.6"
           - "1"
           - "nightly"
-        julia_arch: [x64, x86]
-        exclude:
-          - os: macos-13
-            julia_arch: x86
+        julia_arch: [x64]
 
       fail-fast: false
     env:
@@ -101,6 +99,8 @@ jobs:
       JULIA_MPI_TEST_ABI: OpenMPI
       OMPI_MCA_btl_base_warn_component_unused: 0
       OMPI_MCA_rmaps_base_oversubscribe: true
+      # Allow oversubscription
+      PRTE_MCA_rmaps_default_mapping_policy: :oversubscribe
 
     runs-on: ${{ matrix.os }}
 
@@ -108,11 +108,11 @@ jobs:
     - name: Checkout
       uses: actions/checkout@v4
 
-    - uses: julia-actions/setup-julia@v1
+    - uses: julia-actions/setup-julia@v2
       with:
         arch: ${{ matrix.julia_arch }}
         version: ${{ matrix.julia_version }}
-    - uses: julia-actions/cache@v1
+    - uses: julia-actions/cache@v2
 
     - name: add MPIPreferences
       shell: julia --color=yes --project=. {0}
@@ -167,12 +167,12 @@ jobs:
       run: |
         brew install "${{ matrix.mpi }}"
 
-    - uses: julia-actions/setup-julia@v1
+    - uses: julia-actions/setup-julia@v2
       with:
         version: ${{ matrix.julia_version }}
         arch: ${{ matrix.julia_arch }}
 
-    - uses: julia-actions/cache@v1
+    - uses: julia-actions/cache@v2
     - name: add MPIPreferences
       shell: julia --color=yes --project=. {0}
       run: |
@@ -223,11 +223,11 @@ jobs:
       env:
         MPI: ${{ matrix.mpi }}
 
-    - uses: julia-actions/setup-julia@v1
+    - uses: julia-actions/setup-julia@v2
       with:
         version: ${{ matrix.julia_version }}
 
-    - uses: julia-actions/cache@v1
+    - uses: julia-actions/cache@v2
     - name: add MPIPreferences
       shell: julia --color=yes --project=. {0}
       run: |
@@ -266,11 +266,11 @@ jobs:
     - name: Checkout
       uses: actions/checkout@v4
 
-    - uses: julia-actions/setup-julia@v1
+    - uses: julia-actions/setup-julia@v2
       with:
         version: "1"
 
-    - uses: julia-actions/cache@v1
+    - uses: julia-actions/cache@v2
     - name: add MPIPreferences
       shell: julia --color=yes --project=. {0}
       run: |
@@ -309,11 +309,11 @@ jobs:
       run: msmpisetup.exe -unattend -minimal
       shell: cmd
 
-    - uses: julia-actions/setup-julia@v1
+    - uses: julia-actions/setup-julia@v2
       with:
         version: ${{ matrix.julia_version }}
 
-    - uses: julia-actions/cache@v1
+    - uses: julia-actions/cache@v2
     - name: add MPIPreferences
       shell: julia --color=yes --project=. {0}
       run: |
@@ -365,11 +365,11 @@ jobs:
     - name: Checkout
       uses: actions/checkout@v4
 
-    - uses: julia-actions/setup-julia@v1
+    - uses: julia-actions/setup-julia@v2
       with:
         version: ${{ matrix.julia_version }}
 
-    - uses: julia-actions/cache@v1
+    - uses: julia-actions/cache@v2
 
     - name: add MPIPreferences
       shell: julia --color=yes --project=. {0}
@@ -438,11 +438,11 @@ jobs:
       env:
         MPIWrapper: ${{matrix.MPIWrapper}}
 
-    - uses: julia-actions/setup-julia@v1
+    - uses: julia-actions/setup-julia@v2
       with:
         version: ${{ matrix.julia_version }}
 
-    - uses: julia-actions/cache@v1
+    - uses: julia-actions/cache@v2
 
     - name: add MPIPreferences
       shell: julia --color=yes --project=. {0}
@@ -505,11 +505,11 @@ jobs:
       env:
         MPIWrapper: ${{matrix.MPIWrapper}}
 
-    - uses: julia-actions/setup-julia@v1
+    - uses: julia-actions/setup-julia@v2
       with:
         version: ${{ matrix.julia_version }}
 
-    - uses: julia-actions/cache@v1
+    - uses: julia-actions/cache@v2
 
     - name: add MPIPreferences
       shell: julia --color=yes --project=. {0}
@@ -557,60 +557,11 @@ jobs:
     - name: Checkout
       uses: actions/checkout@v4
 
-    - uses: julia-actions/setup-julia@v1
-      with:
-        version: "1"
-
-    - uses: julia-actions/cache@v1
-    - name: add MPIPreferences
-      shell: julia --color=yes --project=. {0}
-      run: |
-        using Pkg
-        Pkg.develop(path="lib/MPIPreferences")
-        Pkg.precompile()
-
-    - name: use system MPI
-      shell: julia --color=yes --project=. {0}
-      run: |
-        using MPIPreferences
-        MPIPreferences.use_system_binary()
-
-    - uses: julia-actions/julia-runtest@v1
-
-
-  # TODO: Can remove this once we can use OpenMPI_jll v5
-  test-spack-openmpi5:
-    timeout-minutes: 20
-    strategy:
-      matrix:
-        include:
-          - container: ghcr.io/juliaparallel/github-actions-buildcache:openmpi5-jq
-            name: "OpenMPI 5.0.2"
-
-      fail-fast: false
-
-    runs-on: ubuntu-22.04
-    container: ${{ matrix.container }}
-    name: "Test ${{ matrix.name }}"
-
-    env:
-      JULIA_MPI_TEST_BINARY: system
-      JULIA_MPI_TEST_ABI: OpenMPI
-      # Allow running as root
-      OMPI_ALLOW_RUN_AS_ROOT: 1
-      OMPI_ALLOW_RUN_AS_ROOT_CONFIRM: 1
-      # Allow oversubscription
-      PRTE_MCA_rmaps_default_mapping_policy: :oversubscribe
-
-    steps:
-    - name: Checkout
-      uses: actions/checkout@v4
-
-    - uses: julia-actions/setup-julia@v1
+    - uses: julia-actions/setup-julia@v2
       with:
         version: "1"
 
-    - uses: julia-actions/cache@v1
+    - uses: julia-actions/cache@v2
     - name: add MPIPreferences
       shell: julia --color=yes --project=. {0}
       run: |
diff --git a/test/Project.toml b/test/Project.toml
index 21702f8..52ccb43 100644
--- a/test/Project.toml
+++ b/test/Project.toml
@@ -6,14 +6,17 @@ Random = "9a3f8284-a2c9-5f02-9a11-845980a1fd5c"
 TOML = "fa267f1f-6049-4f14-aa54-33bafae1ed76"
 Test = "8dfed614-e22c-5e08-85e1-65c5234f0b40"
 
-[extras]
-AMDGPU = "21141c5a-9bdb-4563-92ae-f87d6854732e"
-CUDA = "052768ef-5323-5732-b1bb-66c8b64840ba"
-
 [weakdeps]
 AMDGPU = "21141c5a-9bdb-4563-92ae-f87d6854732e"
 CUDA = "052768ef-5323-5732-b1bb-66c8b64840ba"
 
 [compat]
-AMDGPU = "0.6, 0.7, 0.8"
+AMDGPU = "0.6, 0.7, 0.8, 0.9, 1"
 CUDA = "3, 4, 5"
+DoubleFloats = "1.4"
+MPIPreferences = "0.1"
+TOML = "< 0.0.1, 1.0"
+
+[extras]
+AMDGPU = "21141c5a-9bdb-4563-92ae-f87d6854732e"
+CUDA = "052768ef-5323-5732-b1bb-66c8b64840ba"
diff --git a/test/mpiexecjl.jl b/test/mpiexecjl.jl
index 4d40bef..5e378ba 100644
--- a/test/mpiexecjl.jl
+++ b/test/mpiexecjl.jl
@@ -15,25 +15,21 @@ using MPI
         # Test a run of mpiexec
         nprocs_str = get(ENV, "JULIA_MPI_TEST_NPROCS", "")
         nprocs = nprocs_str == "" ? clamp(Sys.CPU_THREADS, 2, 4) : parse(Int, nprocs_str)
-        mpiexecjl = joinpath(dir, "mpiexecjl")
+        env = ["JULIA_BINDIR" => Sys.BINDIR]
+        mpiexecjl = addenv(`$(joinpath(dir, "mpiexecjl"))`, env...)
         # `Base.julia_cmd()` ensures keeping consistent flags when running subprocesses.
         julia = Base.julia_cmd()
         example = joinpath(@__DIR__, "..", "docs", "examples", "01-hello.jl")
-        env = ["JULIA_BINDIR" => Sys.BINDIR]
-        p = withenv(env...) do
-            run(`$(mpiexecjl) -n $(nprocs) --project=$(dir) $(julia) --startup-file=no -q $(example)`)
-        end
+        p = run(`$(mpiexecjl) -n $(nprocs) --project=$(dir) $(julia) --startup-file=no -q $(example)`)
         @test success(p)
         # Test help messages
         for help_flag in ("-h", "--help")
-            help_message = withenv(env...) do
-                read(`$(mpiexecjl) --project=$(dir) --help`, String)
-            end
+            help_message = read(`$(mpiexecjl) --project=$(dir) --help`, String)
             @test occursin(r"Usage:.*MPIEXEC_ARGUMENTS", help_message)
         end
         # Without arguments, or only with the `--project` option, the wrapper will fail
-        @test !withenv(() -> success(`$(mpiexecjl) --project=$(dir)`), env...)
-        @test !withenv(() -> success(`$(mpiexecjl)`), env...)
+        @test !success(`$(mpiexecjl) --project=$(dir)`)
+        @test !success(mpiexecjl)
         # Test that the wrapper exits with the same exit code as the MPI process
         exit_code = 10
         p = run(`$(mpiexecjl) -n $(nprocs) --project=$(dir) $(julia) --startup-file=no -e "exit($(exit_code))"`; wait=false)
diff --git a/test/runtests.jl b/test/runtests.jl
index 99b7d18..2a2fdd7 100644
--- a/test/runtests.jl
+++ b/test/runtests.jl
@@ -38,6 +38,11 @@ using DoubleFloats
     end
     CUDA.precompile_runtime()
     ArrayType = CUDA.CuArray
+
+    @info """
+    Running CUDA tests. Ensure that your MPI implementation is
+    CUDA-aware using `MPI.has_cuda` before reporting issues.
+    """
 elseif backend_name == "AMDGPU"
     Pkg.add("AMDGPU")
     ENV["JULIA_MPI_TEST_ARRAYTYPE"] = "ROCArray"
diff --git a/test/test_basic.jl b/test/test_basic.jl
index 495c9f8..3da0139 100644
--- a/test/test_basic.jl
+++ b/test/test_basic.jl
@@ -8,10 +8,22 @@ MPI.Init()
 
 @test MPI.has_cuda() isa Bool
 
-if get(ENV,"JULIA_MPI_TEST_ARRAYTYPE","") == "CuArray"
+if get(ENV, "JULIA_MPI_TEST_ARRAYTYPE", "") == "CuArray"
     @test MPI.has_cuda()
 end
 
+@test MPI.has_rocm() isa Bool
+
+if get(ENV, "JULIA_MPI_TEST_ARRAYTYPE", "") == "ROCArray"
+    @test MPI.has_rocm()
+end
+
+@test MPI.has_gpu() isa Bool
+
+if get(ENV, "JULIA_MPI_TEST_ARRAYTYPE", "") == "CuArray" || get(ENV, "JULIA_MPI_TEST_ARRAYTYPE", "") == "ROCArray"
+    @test MPI.has_gpu()
+end
+
 @test !MPI.Finalized()
 MPI.Finalize()
 @test MPI.Finalized()
diff --git a/test/test_datatype.jl b/test/test_datatype.jl
index c966a96..ab8385d 100644
--- a/test/test_datatype.jl
+++ b/test/test_datatype.jl
@@ -87,18 +87,17 @@ end
 primitive type Primitive16 16 end
 primitive type Primitive24 24 end
 primitive type Primitive80 80 end
+primitive type Primitive104 104 end
+primitive type Primitive136 136 end
 
-@testset for PrimitiveType in (Primitive16, Primitive24, Primitive80)
+@testset for PrimitiveType in (Primitive16, Primitive24, Primitive80, Primitive104, Primitive136)
     sz = sizeof(PrimitiveType)
     al = Base.datatype_alignment(PrimitiveType)
     @test MPI.Types.extent(MPI.Datatype(PrimitiveType)) == (0, cld(sz,al)*al)
 
-    if VERSION < v"1.3" && PrimitiveType == Primitive80
-        # alignment is broken on earlier Julia versions
-        continue
-    end
+    conv = sizeof(PrimitiveType) <= sizeof(UInt128) ? Core.Intrinsics.trunc_int : Core.Intrinsics.sext_int
 
-    arr = [Core.Intrinsics.trunc_int(PrimitiveType, UInt128(comm_rank + i)) for i = 1:4]
+    arr = [conv(PrimitiveType, UInt128(comm_rank + i)) for i = 1:4]
     arr_recv = Array{PrimitiveType}(undef,4)
 
     recv_req = MPI.Irecv!(arr_recv, src, 2, MPI.COMM_WORLD)
@@ -106,7 +105,7 @@ primitive type Primitive80 80 end
 
     MPI.Waitall([recv_req, send_req])
 
-    @test arr_recv == [Core.Intrinsics.trunc_int(PrimitiveType, UInt128(src + i)) for i = 1:4]
+    @test arr_recv == [conv(PrimitiveType, UInt128(src + i)) for i = 1:4]
 end
 
 @testset "packed non-aligned tuples" begin
diff --git a/test/test_reduce.jl b/test/test_reduce.jl
index dbf0c6e..6ad5d24 100644
--- a/test/test_reduce.jl
+++ b/test/test_reduce.jl
@@ -59,10 +59,15 @@ if isroot
     @test sum_mesg == sz .* mesg
 end
 
+function my_reduce(x, y)
+    2x+y-x
+end
+MPI.@RegisterOp(my_reduce, Any)
+
 if can_do_closures
-    operators = [MPI.SUM, +, (x,y) -> 2x+y-x]
+    operators = [MPI.SUM, +, my_reduce, (x,y) -> 2x+y-x]
 else
-    operators = [MPI.SUM, +]
+    operators = [MPI.SUM, +, my_reduce]
 end
 
 for T = [Int]
@@ -117,19 +122,17 @@ end
 
 MPI.Barrier( MPI.COMM_WORLD )
 
-if can_do_closures
-    send_arr = [Double64(i)/10 for i = 1:10]
-
-    result = MPI.Reduce(send_arr, +, MPI.COMM_WORLD; root=root)
-    if rank == root
-        @test result ≈ [Double64(sz*i)/10 for i = 1:10] rtol=sz*eps(Double64)
-    else
-        @test result === nothing
-    end
+send_arr = [Double64(i)/10 for i = 1:10]
 
-    MPI.Barrier( MPI.COMM_WORLD )
+result = MPI.Reduce(send_arr, +, MPI.COMM_WORLD; root=root)
+if rank == root
+    @test result ≈ [Double64(sz*i)/10 for i = 1:10] rtol=sz*eps(Double64)
+else
+    @test result === nothing
 end
 
+MPI.Barrier( MPI.COMM_WORLD )
+
 GC.gc()
 MPI.Finalize()
 @test MPI.Finalized()