Repository: rapidsai/rapidsmpf
Pull request #260: Optional MPI support 
Author: login: madsbk
Labels: name: improvement
description: Improves an existing functionality
name: non-breaking
description: Introduces a non-breaking change
Created: 2025-05-08T13:10:08Z | Merged: 2025-05-09T13:17:12Z

Closes https://github.com/rapidsai/rapidsmpf/issues/219

Locally, this works for me in a no-mpi conda env:
```
./build.sh -v -g  --cmake-args=\"-DBUILD_MPI_SUPPORT=OFF -DBUILD_TESTS=OFF -DBUILD_BENCHMARKS=OFF -DBUILD_EXAMPLES=OFF\" librapidsmpf rapidsmpf && pytest python/rapidsmpf/rapidsmpf/tests/test_dask.py -vsx
```


Diff:
diff --git a/cpp/CMakeLists.txt b/cpp/CMakeLists.txt
index fe4c0282b..ff3bee483 100644
--- a/cpp/CMakeLists.txt
+++ b/cpp/CMakeLists.txt
@@ -33,18 +33,32 @@ rapids_cmake_build_type(Release)
 # ##################################################################################################
 # * build options ---------------------------------------------------------------------------------
 
+option(BUILD_MPI_SUPPORT "Build RapidsMPF with MPI support" ON)
 option(BUILD_TESTS "Configure CMake to build tests" ON)
 option(BUILD_BENCHMARKS "Configure CMake to build benchmarks" ON)
 option(BUILD_EXAMPLES "Configure CMake to build examples" ON)
-option(BUILD_SHARED_LIBS "Build rapidsmpf shared library" ON)
+option(BUILD_SHARED_LIBS "Build RapidsMPF shared library" ON)
 # cudart can be statically linked or dynamically linked. The python ecosystem wants dynamic linking
 option(CUDA_STATIC_RUNTIME "Statically link the CUDA runtime" OFF)
 option(RAPIDSMPF_STATIC_LINTERS "Enable static linters (e.g. clang-tidy) during compilation" OFF)
 
+set(RAPIDSMPF_HAVE_MPI ${BUILD_MPI_SUPPORT})
 set(RAPIDSMPF_BUILD_TESTS ${BUILD_TESTS})
 set(RAPIDSMPF_BUILD_BENCHMARKS ${BUILD_BENCHMARKS})
 set(RAPIDSMPF_BUILD_EXAMPLES ${BUILD_EXAMPLES})
 
+if(NOT RAPIDSMPF_HAVE_MPI)
+  if(RAPIDSMPF_BUILD_TESTS)
+    message(FATAL_ERROR "BUILD_TESTS cannot be ON when BUILD_MPI_SUPPORT is OFF")
+  endif()
+  if(RAPIDSMPF_BUILD_BENCHMARKS)
+    message(FATAL_ERROR "BUILD_BENCHMARKS cannot be ON when BUILD_MPI_SUPPORT is OFF")
+  endif()
+  if(RAPIDSMPF_BUILD_EXAMPLES)
+    message(FATAL_ERROR "BUILD_EXAMPLES cannot be ON when BUILD_MPI_SUPPORT is OFF")
+  endif()
+endif()
+
 # ##################################################################################################
 # * conda environment -----------------------------------------------------------------------------
 rapids_cmake_support_conda_env(conda_env MODIFY_PREFIX_PATH)
@@ -71,11 +85,6 @@ rapids_find_package(
 include(../cmake/thirdparty/get_nvtx.cmake)
 include(../cmake/thirdparty/get_rmm.cmake)
 include(../cmake/thirdparty/get_cudf.cmake)
-rapids_find_package(
-  MPI REQUIRED
-  BUILD_EXPORT_SET rapidsmpf-exports
-  INSTALL_EXPORT_SET rapidsmpf-exports
-)
 rapids_find_package(
   UCX REQUIRED
   BUILD_EXPORT_SET rapidsmpf-exports
@@ -86,6 +95,13 @@ rapids_find_package(
   BUILD_EXPORT_SET rapidsmpf-exports
   INSTALL_EXPORT_SET rapidsmpf-exports
 )
+if(RAPIDSMPF_HAVE_MPI)
+  rapids_find_package(
+    MPI REQUIRED
+    BUILD_EXPORT_SET rapidsmpf-exports
+    INSTALL_EXPORT_SET rapidsmpf-exports
+  )
+endif()
 if(RAPIDSMPF_BUILD_TESTS)
   include(../cmake/thirdparty/get_gtest.cmake)
 endif()
@@ -94,6 +110,13 @@ endif()
 # * library targets --------------------------------------------------------------------------------
 
 file(GLOB SOURCES "src/*.cpp" "src/shuffler/*.cpp" "src/communicator/*.cpp" "src/buffer/*.cpp")
+
+# Without MPI support, remove the MPI communicator from the source list.
+if(NOT RAPIDSMPF_HAVE_MPI)
+  list(REMOVE_ITEM SOURCES "${CMAKE_CURRENT_SOURCE_DIR}/src/communicator/mpi.cpp")
+  list(REMOVE_ITEM SOURCES "${CMAKE_CURRENT_SOURCE_DIR}/src/communicator/ucxx_utils.cpp")
+endif()
+
 add_library(rapidsmpf ${SOURCES})
 
 set_target_properties(
@@ -125,13 +148,10 @@ target_include_directories(
 target_link_libraries(
   rapidsmpf
   PUBLIC rmm::rmm cudf::cudf ucxx::ucxx
-  PRIVATE MPI::MPI_C
+  PRIVATE $<TARGET_NAME_IF_EXISTS:MPI::MPI_C> $<TARGET_NAME_IF_EXISTS:conda_env>
 )
 
-# Add Conda library, and include paths if specified
-if(TARGET conda_env)
-  target_link_libraries(rapidsmpf PRIVATE conda_env)
-endif()
+target_compile_definitions(rapidsmpf PUBLIC $<$<BOOL:${RAPIDSMPF_HAVE_MPI}>:RAPIDSMPF_HAVE_MPI>)
 
 rapids_cuda_set_runtime(rapidsmpf USE_STATIC ${CUDA_STATIC_RUNTIME})
 
@@ -234,12 +254,16 @@ RAPIDSMPF.
     ]=]
 )
 
+# Export `RAPIDSMPF_HAVE_MPI` to let downstream projects know if MPI is supported.
+set(final_code_string "set(RAPIDSMPF_HAVE_MPI [=[${RAPIDSMPF_HAVE_MPI}]=])")
+
 rapids_export(
   INSTALL rapidsmpf
   EXPORT_SET rapidsmpf-exports
   GLOBAL_TARGETS rapidsmpf
   NAMESPACE rapidsmpf::
   DOCUMENTATION doc_string
+  FINAL_CODE_BLOCK final_code_string
 )
 
 # ##################################################################################################
@@ -251,4 +275,5 @@ rapids_export(
   GLOBAL_TARGETS rapidsmpf
   NAMESPACE rapidsmpf::
   DOCUMENTATION doc_string
+  FINAL_CODE_BLOCK final_code_string
 )
diff --git a/cpp/benchmarks/CMakeLists.txt b/cpp/benchmarks/CMakeLists.txt
index 8891bb24a..8f0692977 100644
--- a/cpp/benchmarks/CMakeLists.txt
+++ b/cpp/benchmarks/CMakeLists.txt
@@ -20,8 +20,8 @@ target_compile_options(
                         "$<$<COMPILE_LANGUAGE:CUDA>:${RAPIDSMPF_CUDA_FLAGS}>"
 )
 target_link_libraries(
-  bench_shuffle PRIVATE rapidsmpf::rapidsmpf MPI::MPI_C ucxx::ucxx ucx::ucp
-                        $<TARGET_NAME_IF_EXISTS:conda_env>
+  bench_shuffle PRIVATE rapidsmpf::rapidsmpf ucxx::ucxx ucx::ucp
+                        $<TARGET_NAME_IF_EXISTS:MPI::MPI_C> $<TARGET_NAME_IF_EXISTS:conda_env>
 )
 install(
   TARGETS bench_shuffle
@@ -46,7 +46,7 @@ target_compile_options(
                      "$<$<COMPILE_LANGUAGE:CUDA>:${RAPIDSMPF_CUDA_FLAGS}>"
 )
 target_link_libraries(
-  bench_comm PRIVATE rapidsmpf::rapidsmpf MPI::MPI_C ucxx::ucxx ucx::ucp
+  bench_comm PRIVATE rapidsmpf::rapidsmpf ucxx::ucxx ucx::ucp $<TARGET_NAME_IF_EXISTS:MPI::MPI_C>
                      $<TARGET_NAME_IF_EXISTS:conda_env>
 )
 install(
diff --git a/cpp/examples/CMakeLists.txt b/cpp/examples/CMakeLists.txt
index 7e4e8678f..02b471aca 100644
--- a/cpp/examples/CMakeLists.txt
+++ b/cpp/examples/CMakeLists.txt
@@ -19,8 +19,8 @@ target_compile_options(
                           "$<$<COMPILE_LANGUAGE:CUDA>:${RAPIDSMPF_CUDA_FLAGS}>"
 )
 target_link_libraries(
-  example_shuffle PRIVATE rapidsmpf::rapidsmpf MPI::MPI_C ucxx::ucxx ucx::ucp
-                          $<TARGET_NAME_IF_EXISTS:conda_env>
+  example_shuffle PRIVATE rapidsmpf::rapidsmpf ucxx::ucxx ucx::ucp
+                          $<TARGET_NAME_IF_EXISTS:MPI::MPI_C> $<TARGET_NAME_IF_EXISTS:conda_env>
 )
 install(
   TARGETS example_shuffle
diff --git a/cpp/include/rapidsmpf/communicator/communicator.hpp b/cpp/include/rapidsmpf/communicator/communicator.hpp
index 3a5a34733..70446e22a 100644
--- a/cpp/include/rapidsmpf/communicator/communicator.hpp
+++ b/cpp/include/rapidsmpf/communicator/communicator.hpp
@@ -510,6 +510,16 @@ class Communicator {
     [[nodiscard]] virtual std::string str() const = 0;
 };
 
+/// @brief Whether RapidsMPF was built with the UCXX Communicator.
+constexpr bool COMM_HAVE_UCXX = true;
+
+/// @brief Whether RapidsMPF was built with the MPI Communicator.
+#ifdef RAPIDSMPF_HAVE_MPI
+constexpr bool COMM_HAVE_MPI = true;
+#else
+constexpr bool COMM_HAVE_MPI = false;
+#endif
+
 /**
  * @brief Overloads the stream insertion operator for the Communicator class.
  *
diff --git a/cpp/src/shuffler/shuffler.cpp b/cpp/src/shuffler/shuffler.cpp
index 90e6a9f7d..513a68923 100644
--- a/cpp/src/shuffler/shuffler.cpp
+++ b/cpp/src/shuffler/shuffler.cpp
@@ -7,8 +7,6 @@
 #include <stdexcept>
 #include <utility>
 
-#include <mpi.h>
-
 #include <cudf/concatenate.hpp>
 #include <cudf/detail/contiguous_split.hpp>  // `cudf::detail::pack` (stream ordered version)
 
diff --git a/cpp/tests/CMakeLists.txt b/cpp/tests/CMakeLists.txt
index 0db927847..0f07b2146 100644
--- a/cpp/tests/CMakeLists.txt
+++ b/cpp/tests/CMakeLists.txt
@@ -58,8 +58,16 @@ target_compile_options(
                     "$<$<COMPILE_LANGUAGE:CUDA>:${RAPIDSMPF_CUDA_FLAGS}>"
 )
 target_link_libraries(
-  mpi_tests PRIVATE rapidsmpf::rapidsmpf cudf::cudftestutil cudf::cudftestutil_impl GTest::gmock
-                    GTest::gtest MPI::MPI_C ucxx::ucxx ucx::ucp $<TARGET_NAME_IF_EXISTS:conda_env>
+  mpi_tests
+  PRIVATE rapidsmpf::rapidsmpf
+          cudf::cudftestutil
+          cudf::cudftestutil_impl
+          GTest::gmock
+          GTest::gtest
+          ucxx::ucxx
+          ucx::ucp
+          $<TARGET_NAME_IF_EXISTS:MPI::MPI_C>
+          $<TARGET_NAME_IF_EXISTS:conda_env>
 )
 disable_sign_conversion_warning(mpi_tests)
 
@@ -83,8 +91,16 @@ target_compile_options(
                      "$<$<COMPILE_LANGUAGE:CUDA>:${RAPIDSMPF_CUDA_FLAGS}>"
 )
 target_link_libraries(
-  ucxx_tests PRIVATE rapidsmpf::rapidsmpf cudf::cudftestutil cudf::cudftestutil_impl GTest::gmock
-                     GTest::gtest MPI::MPI_C ucxx::ucxx ucx::ucp $<TARGET_NAME_IF_EXISTS:conda_env>
+  ucxx_tests
+  PRIVATE rapidsmpf::rapidsmpf
+          cudf::cudftestutil
+          cudf::cudftestutil_impl
+          GTest::gmock
+          GTest::gtest
+          ucxx::ucxx
+          ucx::ucp
+          $<TARGET_NAME_IF_EXISTS:MPI::MPI_C>
+          $<TARGET_NAME_IF_EXISTS:conda_env>
 )
 disable_sign_conversion_warning(ucxx_tests)
 
diff --git a/python/rapidsmpf/rapidsmpf/buffer/CMakeLists.txt b/python/rapidsmpf/rapidsmpf/buffer/CMakeLists.txt
index 5737438bb..a7fc31fbd 100644
--- a/python/rapidsmpf/rapidsmpf/buffer/CMakeLists.txt
+++ b/python/rapidsmpf/rapidsmpf/buffer/CMakeLists.txt
@@ -8,5 +8,5 @@ set(cython_modules buffer.pyx packed_data.pyx resource.pyx spill_manager.pyx)
 rapids_cython_create_modules(
   CXX
   SOURCE_FILES "${cython_modules}"
-  LINKED_LIBRARIES rapidsmpf::rapidsmpf MPI::MPI_CXX ucxx::ucxx ucx::ucp
+  LINKED_LIBRARIES rapidsmpf::rapidsmpf ucxx::ucxx ucx::ucp $<TARGET_NAME_IF_EXISTS:MPI::MPI_C>
 )
diff --git a/python/rapidsmpf/rapidsmpf/communicator/CMakeLists.txt b/python/rapidsmpf/rapidsmpf/communicator/CMakeLists.txt
index f9771e232..d659d02dd 100644
--- a/python/rapidsmpf/rapidsmpf/communicator/CMakeLists.txt
+++ b/python/rapidsmpf/rapidsmpf/communicator/CMakeLists.txt
@@ -4,8 +4,8 @@
 # =================================================================================
 
 set(modules_only_rapidsmpf communicator.pyx)
-set(modules_need_mpi mpi.pyx)
 set(modules_need_ucxx ucxx.pyx)
+set(modules_need_mpi mpi.pyx)
 
 rapids_cython_create_modules(
   CXX
@@ -13,14 +13,16 @@ rapids_cython_create_modules(
   LINKED_LIBRARIES rapidsmpf::rapidsmpf
 )
 
-rapids_cython_create_modules(
-  CXX
-  SOURCE_FILES "${modules_need_mpi}"
-  LINKED_LIBRARIES rapidsmpf::rapidsmpf PRIVATE MPI::MPI_CXX
-)
-
 rapids_cython_create_modules(
   CXX
   SOURCE_FILES "${modules_need_ucxx}"
   LINKED_LIBRARIES rapidsmpf::rapidsmpf PRIVATE ucxx::ucxx ucxx::python ucx::ucp
 )
+
+if(RAPIDSMPF_HAVE_MPI)
+  rapids_cython_create_modules(
+    CXX
+    SOURCE_FILES "${modules_need_mpi}"
+    LINKED_LIBRARIES rapidsmpf::rapidsmpf PRIVATE MPI::MPI_CXX
+  )
+endif()
diff --git a/python/rapidsmpf/rapidsmpf/communicator/__init__.py b/python/rapidsmpf/rapidsmpf/communicator/__init__.py
index bb232a363..e5a9bd6b5 100644
--- a/python/rapidsmpf/rapidsmpf/communicator/__init__.py
+++ b/python/rapidsmpf/rapidsmpf/communicator/__init__.py
@@ -1,3 +1,20 @@
 # SPDX-FileCopyrightText: Copyright (c) 2025, NVIDIA CORPORATION & AFFILIATES.
 # SPDX-License-Identifier: Apache-2.0
-"""Submodule for communication abstraction."""
+"""Submodule for communication abstraction (e.g. UCXX and MPI)."""
+
+from __future__ import annotations
+
+from rapidsmpf.communicator.communicator import _available_communicators
+
+COMMUNICATORS = _available_communicators()
+"""Tuple of available communicators.
+
+RapidsMPF includes a collection of communicator backends, available as submodules
+under ``rapidsmpf.communicator.*``. Typically, the Conda distribution includes
+both UCXX and MPI support, while the PIP installation generally supports only UCXX.
+"""
+
+
+__all__ = [
+    "COMMUNICATORS",
+]
diff --git a/python/rapidsmpf/rapidsmpf/communicator/communicator.pxd b/python/rapidsmpf/rapidsmpf/communicator/communicator.pxd
index c30cd3af7..d7874cf10 100644
--- a/python/rapidsmpf/rapidsmpf/communicator/communicator.pxd
+++ b/python/rapidsmpf/rapidsmpf/communicator/communicator.pxd
@@ -12,6 +12,8 @@ cdef class Logger:
 cdef extern from "<rapidsmpf/communicator/communicator.hpp>" namespace \
   "rapidsmpf" nogil:
     ctypedef int32_t Rank
+    cdef const bint COMM_HAVE_UCXX
+    cdef const bint COMM_HAVE_MPI
 
 cdef extern from "<rapidsmpf/communicator/communicator.hpp>" namespace \
   "rapidsmpf::Communicator::Logger" nogil:
diff --git a/python/rapidsmpf/rapidsmpf/communicator/communicator.pyi b/python/rapidsmpf/rapidsmpf/communicator/communicator.pyi
index 85e886973..53a47c2ae 100644
--- a/python/rapidsmpf/rapidsmpf/communicator/communicator.pyi
+++ b/python/rapidsmpf/rapidsmpf/communicator/communicator.pyi
@@ -29,3 +29,5 @@ class Communicator:
     @property
     def logger(self) -> Logger: ...
     def get_str(self) -> str: ...
+
+def _available_communicators() -> tuple[str, ...]: ...
diff --git a/python/rapidsmpf/rapidsmpf/communicator/communicator.pyx b/python/rapidsmpf/rapidsmpf/communicator/communicator.pyx
index 7b205cf76..8b3e62ba7 100644
--- a/python/rapidsmpf/rapidsmpf/communicator/communicator.pyx
+++ b/python/rapidsmpf/rapidsmpf/communicator/communicator.pyx
@@ -186,3 +186,12 @@ cdef class Communicator:
         """
         cdef string s = deref(self._handle).str()
         return s.decode('utf-8')
+
+
+def _available_communicators():
+    ret = []
+    if COMM_HAVE_UCXX:
+        ret.append("ucxx")
+    if COMM_HAVE_MPI:
+        ret.append("mpi")
+    return tuple(ret)
diff --git a/python/rapidsmpf/rapidsmpf/tests/conftest.py b/python/rapidsmpf/rapidsmpf/tests/conftest.py
index 911599dc0..224abccbe 100644
--- a/python/rapidsmpf/rapidsmpf/tests/conftest.py
+++ b/python/rapidsmpf/rapidsmpf/tests/conftest.py
@@ -5,13 +5,11 @@
 from typing import TYPE_CHECKING
 
 import pytest
-from mpi4py import MPI
 
 import rmm.mr
 from rmm.pylibrmm.stream import DEFAULT_STREAM
 
-from rapidsmpf.communicator.mpi import new_communicator
-from rapidsmpf.communicator.testing import ucxx_mpi_setup
+from rapidsmpf.communicator import COMMUNICATORS
 
 if TYPE_CHECKING:
     from collections.abc import Generator
@@ -31,6 +29,10 @@ def _mpi_comm() -> Communicator:
 
     Do not use this fixture directly, use the `comm` fixture instead.
     """
+    from mpi4py import MPI
+
+    from rapidsmpf.communicator.mpi import new_communicator
+
     return new_communicator(MPI.COMM_WORLD)
 
 
@@ -44,6 +46,8 @@ def _ucxx_comm() -> Communicator:
 
     Do not use this fixture directly, use the `ucxx_comm` fixture instead.
     """
+    from rapidsmpf.communicator.testing import ucxx_mpi_setup
+
     return ucxx_mpi_setup(None)
 
 
@@ -52,10 +56,25 @@ def _ucxx_comm() -> Communicator:
 )
 def comm(request: pytest.FixtureRequest) -> Generator[Communicator]:
     """
-    Fixture for a rapidsmpf communicator, scoped for each test.
+    Fixture for a rapidsmpf communicator and setup, scoped for each test.
     """
+    comm_name = request.param
+
+    if "mpi" not in COMMUNICATORS:
+        if comm_name == "mpi":
+            pytest.skip("RapidsMPF not built with MPI support")
+        if comm_name == "ucxx":
+            pytest.skip(
+                "RapidsMPF not built with MPI support, which is "
+                "used to bootstrap this UCXX test"
+            )
+    if "ucxx" not in COMMUNICATORS:
+        pytest.skip("RapidsMPF not built with UCXX support")
+
+    from mpi4py import MPI
+
     MPI.COMM_WORLD.barrier()
-    yield request.getfixturevalue(f"_{request.param}_comm")
+    yield request.getfixturevalue(f"_{comm_name}_comm")
     MPI.COMM_WORLD.barrier()
 
 
diff --git a/python/rapidsmpf/rapidsmpf/tests/test_dask.py b/python/rapidsmpf/rapidsmpf/tests/test_dask.py
index 8e55942c9..6a4a7c834 100644
--- a/python/rapidsmpf/rapidsmpf/tests/test_dask.py
+++ b/python/rapidsmpf/rapidsmpf/tests/test_dask.py
@@ -2,12 +2,13 @@
 # SPDX-License-Identifier: Apache-2.0
 from __future__ import annotations
 
-from typing import TYPE_CHECKING, cast
+from typing import TYPE_CHECKING
 
 import dask
 import dask.dataframe as dd
 import pytest
 
+from rapidsmpf.communicator import COMMUNICATORS
 from rapidsmpf.examples.dask import DaskCudfIntegration
 from rapidsmpf.integrations.dask.core import get_worker_context
 from rapidsmpf.integrations.dask.shuffler import rapidsmpf_shuffle_graph
@@ -24,7 +25,6 @@
     loop,
     loop_in_thread,
 )
-from mpi4py import MPI  # noqa: E402
 
 from rapidsmpf.integrations.dask.core import (  # noqa: E402
     bootstrap_dask_cluster,
@@ -34,15 +34,17 @@
     from distributed.worker import Worker
 
 
-def get_mpi_nsize() -> int:
-    comm = MPI.COMM_WORLD
-    size = comm.Get_size()
-    # without the cast, mypy sees this as Any
-    return cast(int, size)
+def is_running_on_multiple_mpi_nodes() -> bool:
+    if "mpi" not in COMMUNICATORS:
+        return False
+
+    from mpi4py import MPI
+
+    return bool(MPI.COMM_WORLD.Get_size() > 1)
 
 
 pytestmark = pytest.mark.skipif(
-    get_mpi_nsize() > 1,
+    is_running_on_multiple_mpi_nodes(),
     reason="Dask tests should not run with more than one MPI process",
 )
 


Reviews:
user: login: pentschev
state: COMMENTED
author_association: MEMBER
submitted_at: 2025-05-08T13:54:05Z
user: login: madsbk
state: COMMENTED
author_association: MEMBER
submitted_at: 2025-05-08T13:57:52Z
user: login: pentschev
body: LGTM, thanks @madsbk !
state: APPROVED
author_association: MEMBER
submitted_at: 2025-05-08T14:01:15Z
user: login: wence-
body: Some small suggestions
state: COMMENTED
author_association: CONTRIBUTOR
submitted_at: 2025-05-08T16:36:55Z
user: login: madsbk
state: COMMENTED
author_association: MEMBER
submitted_at: 2025-05-08T17:38:41Z
user: login: wence-
state: APPROVED
author_association: CONTRIBUTOR
submitted_at: 2025-05-09T13:15:17Z

Review comments:
diff_hunk: @@ -33,18 +33,32 @@ rapids_cmake_build_type(Release)
 # ##################################################################################################
 # * build options ---------------------------------------------------------------------------------
 
+option(BUILD_MPI_SUPPORT "Build RapidsMPF with MPI support" ON)
 option(BUILD_TESTS "Configure CMake to build tests" ON)
 option(BUILD_BENCHMARKS "Configure CMake to build benchmarks" ON)
 option(BUILD_EXAMPLES "Configure CMake to build examples" ON)
-option(BUILD_SHARED_LIBS "Build rapidsmpf shared library" ON)
+option(BUILD_SHARED_LIBS "Build RapidsMPF shared library" ON)
 # cudart can be statically linked or dynamically linked. The python ecosystem wants dynamic linking
 option(CUDA_STATIC_RUNTIME "Statically link the CUDA runtime" OFF)
 option(RAPIDSMPF_STATIC_LINTERS "Enable static linters (e.g. clang-tidy) during compilation" OFF)
 
+set(RAPIDSMPF_BUILD_MPI_SUPPORT ${BUILD_MPI_SUPPORT})
 set(RAPIDSMPF_BUILD_TESTS ${BUILD_TESTS})
 set(RAPIDSMPF_BUILD_BENCHMARKS ${BUILD_BENCHMARKS})
 set(RAPIDSMPF_BUILD_EXAMPLES ${BUILD_EXAMPLES})
 
+if(NOT RAPIDSMPF_BUILD_MPI_SUPPORT)
+  if(RAPIDSMPF_BUILD_TESTS)
+    message(FATAL_ERROR "BUILD_TESTS cannot be ON when BUILD_MPI_SUPPORT is OFF")
+  endif()
+  if(RAPIDSMPF_BUILD_BENCHMARKS)
+    message(FATAL_ERROR "BUILD_BENCHMARKS cannot be ON when BUILD_MPI_SUPPORT is OFF")
+  endif()
+  if(RAPIDSMPF_BUILD_EXAMPLES)
+    message(FATAL_ERROR "BUILD_EXAMPLES cannot be ON when BUILD_MPI_SUPPORT is OFF")
+  endif()
+endif()
path: cpp/CMakeLists.txt
user: login: pentschev
body: I forgot we can't even run any of that without MPI. I'll work on a small launcher for UCXX benchmarks at least.
created_at: 2025-05-08T13:15:41Z
author_association: MEMBER
diff_hunk: @@ -20,8 +20,8 @@ target_compile_options(
                         "$<$<COMPILE_LANGUAGE:CUDA>:${RAPIDSMPF_CUDA_FLAGS}>"
 )
 target_link_libraries(
-  bench_shuffle PRIVATE rapidsmpf::rapidsmpf MPI::MPI_C ucxx::ucxx ucx::ucp
-                        $<TARGET_NAME_IF_EXISTS:conda_env>
+  bench_shuffle PRIVATE rapidsmpf::rapidsmpf ucxx::ucxx ucx::ucp
+                        $<TARGET_NAME_IF_EXISTS:MPI::MPI_C> $<TARGET_NAME_IF_EXISTS:conda_env>
path: cpp/benchmarks/CMakeLists.txt
user: login: pentschev
body: Not a big deal, but do we even need to check whether MPI exists here, as well as in examples/tests? Currently benchmarks are always disabled if MPI is also disabled, making this check redundant. However, I can see the appeal of leaving this for future use when we have a standalone launcher for UCXX, for example.
created_at: 2025-05-08T13:24:47Z
author_association: MEMBER
diff_hunk: @@ -20,8 +20,8 @@ target_compile_options(
                         "$<$<COMPILE_LANGUAGE:CUDA>:${RAPIDSMPF_CUDA_FLAGS}>"
 )
 target_link_libraries(
-  bench_shuffle PRIVATE rapidsmpf::rapidsmpf MPI::MPI_C ucxx::ucxx ucx::ucp
-                        $<TARGET_NAME_IF_EXISTS:conda_env>
+  bench_shuffle PRIVATE rapidsmpf::rapidsmpf ucxx::ucxx ucx::ucp
+                        $<TARGET_NAME_IF_EXISTS:MPI::MPI_C> $<TARGET_NAME_IF_EXISTS:conda_env>
path: cpp/benchmarks/CMakeLists.txt
user: login: madsbk
body: > I can see the appeal of leaving this for future use when we have a standalone launcher for UCXX

Yeah, that was my thinking
created_at: 2025-05-08T13:57:52Z
author_association: MEMBER
diff_hunk: @@ -94,6 +110,13 @@ endif()
 # * library targets --------------------------------------------------------------------------------
 
 file(GLOB SOURCES "src/*.cpp" "src/shuffler/*.cpp" "src/communicator/*.cpp" "src/buffer/*.cpp")
+
+# Without MPI support, remove the MPI communicator from the source list.
+if(NOT RAPIDSMPF_BUILD_MPI_SUPPORT)
+  list(REMOVE_ITEM SOURCES "${CMAKE_CURRENT_SOURCE_DIR}/src/communicator/mpi.cpp")
+  list(REMOVE_ITEM SOURCES "${CMAKE_CURRENT_SOURCE_DIR}/src/communicator/ucxx_utils.cpp")
+endif()
path: cpp/CMakeLists.txt
user: login: wence-
body: Suggestion: let's start explicitly listing the files and conditionally include, rather than conditionally removing.
created_at: 2025-05-08T16:21:08Z
author_association: CONTRIBUTOR
diff_hunk: @@ -1,3 +1,19 @@
 # SPDX-FileCopyrightText: Copyright (c) 2025, NVIDIA CORPORATION & AFFILIATES.
 # SPDX-License-Identifier: Apache-2.0
-"""Submodule for communication abstraction."""
+"""Submodule for communication abstraction (e.g. UCXX and MPI)."""
+
+from __future__ import annotations
+
+import importlib.util
+
+MPI_SUPPORT: bool = importlib.util.find_spec("rapidsmpf.communicator.mpi") is not None
+"""Whether the MPI communicator (``rapidsmpf.communicator.mpi``) is available.
path: python/rapidsmpf/rapidsmpf/communicator/__init__.py
user: login: wence-
body: Suggestion: Can the C++ API expose data about the compile-time configuration which we can just ask, rather than checking whether a particular compiled module exists or not
created_at: 2025-05-08T16:33:38Z
author_association: CONTRIBUTOR
diff_hunk: @@ -125,13 +148,12 @@ target_include_directories(
 target_link_libraries(
   rapidsmpf
   PUBLIC rmm::rmm cudf::cudf ucxx::ucxx
-  PRIVATE MPI::MPI_C
+  PRIVATE $<TARGET_NAME_IF_EXISTS:MPI::MPI_C> $<TARGET_NAME_IF_EXISTS:conda_env>
 )
 
-# Add Conda library, and include paths if specified
-if(TARGET conda_env)
-  target_link_libraries(rapidsmpf PRIVATE conda_env)
-endif()
+target_compile_definitions(
+  rapidsmpf PUBLIC $<$<BOOL:${RAPIDSMPF_BUILD_MPI_SUPPORT}>:RAPIDSMPF_BUILD_MPI_SUPPORT>
+)
path: cpp/CMakeLists.txt
user: login: wence-
body: Suggestion: call this `RAPIDSMPF_HAVE_MPI` (that's the normal "autoconf" naming).
created_at: 2025-05-08T16:34:33Z
author_association: CONTRIBUTOR
diff_hunk: @@ -234,12 +256,16 @@ RAPIDSMPF.
     ]=]
 )
 
+# Export `RAPIDSMPF_BUILD_MPI_SUPPORT` to let downstream projects know if MPI is supported.
+set(final_code_string "set(RAPIDSMPF_BUILD_MPI_SUPPORT [=[${RAPIDSMPF_BUILD_MPI_SUPPORT}]=])")
path: cpp/CMakeLists.txt
user: login: wence-
body: Again, perhaps `RAPIDSMPF_HAVE_MPI`?
created_at: 2025-05-08T16:35:23Z
author_association: CONTRIBUTOR
diff_hunk: @@ -94,6 +110,13 @@ endif()
 # * library targets --------------------------------------------------------------------------------
 
 file(GLOB SOURCES "src/*.cpp" "src/shuffler/*.cpp" "src/communicator/*.cpp" "src/buffer/*.cpp")
+
+# Without MPI support, remove the MPI communicator from the source list.
+if(NOT RAPIDSMPF_BUILD_MPI_SUPPORT)
+  list(REMOVE_ITEM SOURCES "${CMAKE_CURRENT_SOURCE_DIR}/src/communicator/mpi.cpp")
+  list(REMOVE_ITEM SOURCES "${CMAKE_CURRENT_SOURCE_DIR}/src/communicator/ucxx_utils.cpp")
+endif()
path: cpp/CMakeLists.txt
user: login: madsbk
body: Why? I never understood why we do that in cudf.
created_at: 2025-05-08T17:38:40Z
author_association: MEMBER

PR comments:
user: login: nirandaperera
created_at: 2025-05-08T16:14:09Z
body: @madsbk Should we also introduce a CMake definition (`-DRAPIDSMPF_MPI_ENABLED` or something), then we can branch out in the code itself?
author_association: CONTRIBUTOR
user: login: madsbk
created_at: 2025-05-08T17:47:21Z
body: > @madsbk Should we also introduce a CMake definition (`-DRAPIDSMPF_MPI_ENABLED` or something), then we can branch out in the code itself?


It is done here: https://github.com/rapidsai/rapidsmpf/pull/260/files#r2080076775
author_association: MEMBER
user: login: madsbk
created_at: 2025-05-09T13:17:08Z
body: /merge
author_association: MEMBER