Repository: hpc4cmb/toast (language: python)

Issue:
Redesigning OpenMP target offload loops
Precomputing the maximum interval size to be able to fully collapse the triple loop in most operators leaded to significantly improved computation density on the GPU.


Discussion / hints:

Fix patch:
diff --git a/platforms/linux-llvm.sh b/platforms/linux-llvm.sh
index ff22c5092..e77a76d6a 100755
--- a/platforms/linux-llvm.sh
+++ b/platforms/linux-llvm.sh
@@ -1,18 +1,30 @@
 #!/bin/bash
 
-# Pass extra cmake options to this script, including
-# things like -DCMAKE_INSTALL_PREFIX=/path/to/install, etc.
+# This assumes you have created a virtualenv and installed toast
+# dependencies (e.g. with wheels/install_deps_linux.sh) into that virtual
+# env.  This assumes installation of llvm-17 from LLVM apt sources.
 
-opts="$@"
+venv_path=$(dirname $(dirname $(which python3)))
+
+export LD_LIBRARY_PATH="${venv_path}/lib"
 
 cmake \
-    -DCMAKE_C_COMPILER="clang" \
-    -DCMAKE_CXX_COMPILER="clang++" \
-    -DCMAKE_C_FLAGS="-O3 -g -fPIC -pthread" \
-    -DCMAKE_CXX_FLAGS="-O3 -g -fPIC -pthread -std=c++11" \
-    -DOPENMP_TARGET_FLAGS="-fopenmp -fopenmp-targets=nvptx64 -fopenmp-target-debug=3 --offload-arch=sm_80 " \
-    -DPYTHON_EXECUTABLE:FILEPATH=$(which python3) \
+    -DCMAKE_C_COMPILER="clang-17" \
+    -DCMAKE_CXX_COMPILER="clang++-17" \
+    -DCMAKE_C_FLAGS="-O3 -g -fPIC -pthread -I${venv_path}/include" \
+    -DCMAKE_CXX_FLAGS="-O3 -g -fPIC -pthread -std=c++11 -stdlib=libc++ -I${venv_path}/include" \
     -DUSE_OPENMP_TARGET=TRUE \
-    -DCMAKE_VERBOSE_MAKEFILE:BOOL=ON \
-    ${opts} \
+    -DOPENMP_TARGET_FLAGS="-fopenmp -fopenmp-targets=nvptx64 -fopenmp-target-debug=3 --offload-arch=sm_86 -Wl,-lomp,-lomptarget,-lomptarget.rtl.cuda" \
+    -DPYTHON_EXECUTABLE:FILEPATH=$(which python3) \
+    -DCMAKE_VERBOSE_MAKEFILE=ON \
+    -DCMAKE_LIBRARY_PATH="${venv_path}/lib" \
+    -DBLAS_LIBRARIES="-L${venv_path}/lib -lopenblas -fopenmp -lm -lgfortran" \
+    -DLAPACK_LIBRARIES="-L${venv_path}/lib -lopenblas -fopenmp -lm -lgfortran" \
+    -DAATM_ROOT="${venv_path}" \
+    -DFFTW_ROOT="${venv_path}" \
+    -DSUITESPARSE_INCLUDE_DIR_HINTS="${venv_path}/include" \
+    -DSUITESPARSE_LIBRARY_DIR_HINTS="${venv_path}/lib" \
+    -DTOAST_STATIC_DEPS=ON \
+    -DCMAKE_INSTALL_PREFIX="${venv_path}" \
     ..
+
diff --git a/src/toast/_libtoast/accelerator.cpp b/src/toast/_libtoast/accelerator.cpp
index 4a7329cd1..32a67600c 100644
--- a/src/toast/_libtoast/accelerator.cpp
+++ b/src/toast/_libtoast/accelerator.cpp
@@ -634,11 +634,10 @@ void OmpManager::reset(void * buffer, size_t nbytes, std::string const & name) {
       << "') with " << nbytes << " bytes";
     log.verbose(o.str().c_str());
 
-    # pragma omp target data \
-    map(to: nbytes)
+    # pragma omp target data map(to: nbytes)
     {
-        # pragma omp target teams distribute is_device_ptr(dev_buffer) \
-        parallel for default(shared)
+        # pragma \
+        omp target teams distribute parallel for default(shared) is_device_ptr(dev_buffer)
         for (size_t i = 0; i < nbytes; ++i) {
             ((uint8_t *)dev_buffer)[i] = 0;
         }
@@ -1126,17 +1125,14 @@ void init_accelerator(py::module & m) {
             int dev = omgr.get_device();
             bool offload = !omgr.device_is_host();
 
-            void * dev_raw = omgr.device_ptr(raw);
+            double * dev_raw = omgr.device_ptr(raw);
 
             #ifdef HAVE_OPENMP_TARGET
-            # pragma omp target data
-            {
-                # pragma omp target teams distribute parallel for collapse(2) \
-                is_device_ptr(dev_raw)
-                for (int64_t i = 0; i < n_det; i++) {
-                    for (int64_t j = 0; j < n_samp; j++) {
-                        raw[i * n_samp + j] *= 2.0;
-                    }
+            # pragma \
+            omp target teams distribute parallel for is_device_ptr(dev_raw) collapse(2)
+            for (int64_t i = 0; i < n_det; i++) {
+                for (int64_t j = 0; j < n_samp; j++) {
+                    dev_raw[i * n_samp + j] *= 2.0;
                 }
             }
             #endif // ifdef HAVE_OPENMP_TARGET
@@ -1155,17 +1151,14 @@ void init_accelerator(py::module & m) {
             auto & omgr = OmpManager::get();
             int dev = omgr.get_device();
             bool offload = !omgr.device_is_host();
-            void * dev_raw = omgr.device_ptr(raw);
+            double * dev_raw = omgr.device_ptr(raw);
 
             #ifdef HAVE_OPENMP_TARGET
-            # pragma omp target data
-            {
-                # pragma omp target teams distribute parallel for collapse(2) \
-                is_device_ptr(dev_raw)
-                for (size_t i = 0; i < n_det; i++) {
-                    for (size_t j = 0; j < n_samp; j++) {
-                        raw[i * n_samp + j] *= 2.0;
-                    }
+            # pragma \
+            omp target teams distribute parallel for is_device_ptr(dev_raw) collapse(2)
+            for (size_t i = 0; i < n_det; i++) {
+                for (size_t j = 0; j < n_samp; j++) {
+                    dev_raw[i * n_samp + j] *= 2.0;
                 }
             }
             #endif // ifdef HAVE_OPENMP_TARGET
diff --git a/src/toast/_libtoast/ops_mapmaker_utils.cpp b/src/toast/_libtoast/ops_mapmaker_utils.cpp
index 42767a83a..af6647d1a 100644
--- a/src/toast/_libtoast/ops_mapmaker_utils.cpp
+++ b/src/toast/_libtoast/ops_mapmaker_utils.cpp
@@ -33,7 +33,8 @@ void build_noise_weighted_inner(
     uint8_t shared_mask,
     int64_t n_pix_submap,
     bool use_shared_flags,
-    bool use_det_flags) {
+    bool use_det_flags
+) {
     int32_t w_indx = weight_index[idet];
     int32_t p_indx = pixel_index[idet];
     int32_t f_indx = flag_index[idet];
@@ -62,7 +63,8 @@ void build_noise_weighted_inner(
     if (
         (pixels[off_p] >= 0) &&
         (det_check == 0) &&
-        (shared_check == 0)) {
+        (shared_check == 0)
+    ) {
         // Good data, accumulate
         global_submap = (int64_t)(pixels[off_p] / n_pix_submap);
 
@@ -105,8 +107,8 @@ void init_ops_mapmaker_utils(py::module & m) {
             py::buffer intervals,
             py::buffer shared_flags,
             uint8_t shared_flag_mask,
-            bool use_accel)
-        {
+            bool use_accel
+        ) {
             auto & omgr = OmpManager::get();
             int dev = omgr.get_device();
             bool offload = (!omgr.device_is_host()) && use_accel;
@@ -216,24 +218,35 @@ void init_ops_mapmaker_utils(py::module & m) {
                     }
                 }
 
-                # pragma omp target data map(to : raw_weight_index[0 : n_det], \
-                raw_pixel_index[0 : n_det],                                    \
-                raw_flag_index[0 : n_det],                                     \
-                raw_data_index[0 : n_det],                                     \
-                raw_det_scale[0 : n_det],                                      \
-                raw_global2local[0 : n_global_submap],                         \
-                n_view,                                                        \
-                n_det,                                                         \
-                n_samp,                                                        \
-                max_interval_size,                                             \
-                nnz,                                                           \
-                n_pix_submap,                                                  \
-                det_flag_mask,                                                 \
-                shared_flag_mask,                                              \
-                use_shared_flags,                                              \
-                use_det_flags)
+                # pragma omp target data map(          \
+                to : raw_weight_index[0 : n_det],      \
+                raw_pixel_index[0 : n_det],            \
+                raw_flag_index[0 : n_det],             \
+                raw_data_index[0 : n_det],             \
+                raw_det_scale[0 : n_det],              \
+                raw_global2local[0 : n_global_submap], \
+                n_view,                                \
+                n_det,                                 \
+                n_samp,                                \
+                max_interval_size,                     \
+                nnz,                                   \
+                n_pix_submap,                          \
+                det_flag_mask,                         \
+                shared_flag_mask,                      \
+                use_shared_flags,                      \
+                use_det_flags                          \
+                )
                 {
-                    # pragma omp target teams distribute parallel for collapse(3)
+                    # pragma omp target teams distribute parallel for collapse(3) \
+                    is_device_ptr(                                                \
+                    dev_pixels,                                                   \
+                    dev_weights,                                                  \
+                    dev_det_data,                                                 \
+                    dev_det_flags,                                                \
+                    dev_intervals,                                                \
+                    dev_shared_flags,                                             \
+                    dev_zmap                                                      \
+                    )
                     for (int64_t idet = 0; idet < n_det; idet++) {
                         for (int64_t iview = 0; iview < n_view; iview++) {
                             for (int64_t isamp = 0; isamp < max_interval_size; isamp++) {
diff --git a/src/toast/_libtoast/ops_noise_weight.cpp b/src/toast/_libtoast/ops_noise_weight.cpp
index ea8196b60..7a5c7352e 100644
--- a/src/toast/_libtoast/ops_noise_weight.cpp
+++ b/src/toast/_libtoast/ops_noise_weight.cpp
@@ -15,8 +15,8 @@ void init_ops_noise_weight(py::module & m) {
             py::buffer data_index,
             py::buffer intervals,
             py::buffer detector_weights,
-            bool use_accel)
-        {
+            bool use_accel
+        ) {
             auto & omgr = OmpManager::get();
             int dev = omgr.get_device();
             bool offload = (!omgr.device_is_host()) && use_accel;
@@ -60,13 +60,19 @@ void init_ops_noise_weight(py::module & m) {
                     }
                 }
 
-                # pragma omp target data map(to : raw_data_index[0 : n_det], \
-                raw_det_weights[0 : n_det],                                  \
-                n_view,                                                      \
-                n_det,                                                       \
-                n_samp)
+                # pragma omp target data map(   \
+                to : raw_data_index[0 : n_det], \
+                raw_det_weights[0 : n_det],     \
+                n_view,                         \
+                n_det,                          \
+                n_samp                          \
+                )
                 {
-                    # pragma omp target teams distribute parallel for collapse(3)
+                    # pragma omp target teams distribute parallel for collapse(3) \
+                    is_device_ptr(                                                \
+                    dev_det_data,                                                 \
+                    dev_intervals                                                 \
+                    )
                     for (int64_t idet = 0; idet < n_det; idet++) {
                         for (int64_t iview = 0; iview < n_view; iview++) {
                             for (int64_t isamp = 0; isamp < max_interval_size; isamp++) {
diff --git a/src/toast/_libtoast/ops_pixels_healpix.cpp b/src/toast/_libtoast/ops_pixels_healpix.cpp
index 9e9c4bf1b..b12cd49f1 100644
--- a/src/toast/_libtoast/ops_pixels_healpix.cpp
+++ b/src/toast/_libtoast/ops_pixels_healpix.cpp
@@ -45,16 +45,13 @@ void pixels_healpix_qa_rotate(double const * q_in, double const * v_in,
     double z2 = -q_in[2] * q_in[2];
 
     v_out[0] = 2 * ((y2 + z2) * v_in[0] + (xy - zw) * v_in[1] +
-                    (yw + xz) * v_in[2]) +
-               v_in[0];
+                    (yw + xz) * v_in[2]) + v_in[0];
 
     v_out[1] = 2 * ((zw + xy) * v_in[0] + (x2 + z2) * v_in[1] +
-                    (yz - xw) * v_in[2]) +
-               v_in[1];
+                    (yz - xw) * v_in[2]) + v_in[1];
 
     v_out[2] = 2 * ((xz - yw) * v_in[0] + (xw + yz) * v_in[1] +
-                    (x2 + y2) * v_in[2]) +
-               v_in[2];
+                    (x2 + y2) * v_in[2]) + v_in[2];
 
     return;
 }
@@ -205,7 +202,7 @@ void hpix_zphi2ring(int64_t nside, int64_t factor, double phi, int region, doubl
         ip -= longpart;
 
         (*pix) = (region > 0) ? (2 * ir * (ir - 1) + ip)
-                              : (npix - 2 * ir * (ir + 1) + ip);
+            : (npix - 2 * ir * (ir + 1) + ip);
     }
 
     return;
@@ -226,7 +223,8 @@ void pixels_healpix_nest_inner(
     int64_t n_samp,
     int64_t idet,
     uint8_t mask,
-    bool use_flags) {
+    bool use_flags
+) {
     const double zaxis[3] = {0.0, 0.0, 1.0};
     int32_t p_indx = pixel_index[idet];
     int32_t q_indx = quat_index[idet];
@@ -310,8 +308,8 @@ void init_ops_pixels_healpix(py::module & m) {
             int64_t n_pix_submap,
             int64_t nside,
             bool nest,
-            bool use_accel)
-        {
+            bool use_accel
+        ) {
             auto & omgr = OmpManager::get();
             int dev = omgr.get_device();
             bool offload = (!omgr.device_is_host()) && use_accel;
@@ -394,21 +392,30 @@ void init_ops_pixels_healpix(py::module & m) {
                     }
                 }
 
-                # pragma omp target data map(to : raw_pixel_index[0 : n_det], \
-                raw_quat_index[0 : n_det],                                    \
-                n_pix_submap,                                                 \
-                nside,                                                        \
-                factor,                                                       \
-                nest,                                                         \
-                n_view,                                                       \
-                n_det,                                                        \
-                n_samp,                                                       \
-                shared_flag_mask,                                             \
-                use_flags)                                                    \
+                # pragma omp target data map(    \
+                to : raw_pixel_index[0 : n_det], \
+                raw_quat_index[0 : n_det],       \
+                n_pix_submap,                    \
+                nside,                           \
+                factor,                          \
+                nest,                            \
+                n_view,                          \
+                n_det,                           \
+                n_samp,                          \
+                shared_flag_mask,                \
+                use_flags                        \
+                )                                \
                 map(tofrom : raw_hsub[0 : n_submap])
                 {
                     if (nest) {
-                        # pragma omp target teams distribute parallel for collapse(3)
+                        # pragma omp target teams distribute parallel for collapse(3) \
+                        is_device_ptr(                                                \
+                        dev_pixels,                                                   \
+                        dev_quats,                                                    \
+                        dev_flags,                                                    \
+                        dev_intervals,                                                \
+                        dev_utab                                                      \
+                        )
                         for (int64_t idet = 0; idet < n_det; idet++) {
                             for (int64_t iview = 0; iview < n_view; iview++) {
                                 for (int64_t isamp = 0; isamp < max_interval_size;
@@ -443,7 +450,14 @@ void init_ops_pixels_healpix(py::module & m) {
                             }
                         }
                     } else {
-                        # pragma omp target teams distribute parallel for collapse(3)
+                        # pragma omp target teams distribute parallel for collapse(3) \
+                        is_device_ptr(                                                \
+                        dev_pixels,                                                   \
+                        dev_quats,                                                    \
+                        dev_flags,                                                    \
+                        dev_intervals,                                                \
+                        dev_utab                                                      \
+                        )
                         for (int64_t idet = 0; idet < n_det; idet++) {
                             for (int64_t iview = 0; iview < n_view; iview++) {
                                 for (int64_t isamp = 0; isamp < max_interval_size;
diff --git a/src/toast/_libtoast/ops_pointing_detector.cpp b/src/toast/_libtoast/ops_pointing_detector.cpp
index cc8c04039..5c10afc47 100644
--- a/src/toast/_libtoast/ops_pointing_detector.cpp
+++ b/src/toast/_libtoast/ops_pointing_detector.cpp
@@ -40,7 +40,8 @@ void pointing_detector_inner(
     int64_t n_samp,
     int64_t idet,
     uint8_t mask,
-    bool use_flags) {
+    bool use_flags
+) {
     int32_t qidx = q_index[idet];
     double temp_bore[4];
     uint8_t check = 0;
@@ -61,7 +62,8 @@ void pointing_detector_inner(
     pointing_detector_qa_mult(
         temp_bore,
         &(fp[4 * idet]),
-        &(quats[(qidx * 4 * n_samp) + 4 * isamp]));
+        &(quats[(qidx * 4 * n_samp) + 4 * isamp])
+    );
     return;
 }
 
@@ -82,8 +84,8 @@ void init_ops_pointing_detector(py::module & m) {
             py::buffer intervals,
             py::buffer shared_flags,
             uint8_t shared_flag_mask,
-            bool use_accel)
-        {
+            bool use_accel
+        ) {
             auto & omgr = OmpManager::get();
             int dev = omgr.get_device();
             bool offload = (!omgr.device_is_host()) && use_accel;
@@ -145,15 +147,23 @@ void init_ops_pointing_detector(py::module & m) {
                     }
                 }
 
-                # pragma omp target data map(to : raw_focalplane[0 : 4 * n_det], \
-                raw_quat_index[0 : n_det],                                       \
-                shared_flag_mask,                                                \
-                n_view,                                                          \
-                n_det,                                                           \
-                n_samp,                                                          \
-                use_flags)
+                # pragma omp target data map(       \
+                to : raw_focalplane[0 : 4 * n_det], \
+                raw_quat_index[0 : n_det],          \
+                shared_flag_mask,                   \
+                n_view,                             \
+                n_det,                              \
+                n_samp,                             \
+                use_flags                           \
+                )
                 {
-                    # pragma omp target teams distribute parallel for collapse(3)
+                    # pragma omp target teams distribute parallel for collapse(3) \
+                    is_device_ptr(                                                \
+                    dev_boresight,                                                \
+                    dev_quats,                                                    \
+                    dev_flags,                                                    \
+                    dev_intervals                                                 \
+                    )
                     for (int64_t idet = 0; idet < n_det; idet++) {
                         for (int64_t iview = 0; iview < n_view; iview++) {
                             for (int64_t isamp = 0; isamp < max_interval_size; isamp++) {
diff --git a/src/toast/_libtoast/ops_scan_map.cpp b/src/toast/_libtoast/ops_scan_map.cpp
index 3a61131b3..21da3b8e0 100644
--- a/src/toast/_libtoast/ops_scan_map.cpp
+++ b/src/toast/_libtoast/ops_scan_map.cpp
@@ -109,15 +109,18 @@ void register_ops_scan_map(py::module & m, char const * name) {
               std::vector <int64_t> temp_shape(3);
 
               int32_t * raw_pixel_index = extract_buffer <int32_t> (
-                  pixel_index, "pixel_index", 1, temp_shape, {-1});
+                  pixel_index, "pixel_index", 1, temp_shape, {-1}
+              );
               int64_t n_det = temp_shape[0];
 
               int64_t * raw_pixels = extract_buffer <int64_t> (
-                  pixels, "pixels", 2, temp_shape, {-1, -1});
+                  pixels, "pixels", 2, temp_shape, {-1, -1}
+              );
               int64_t n_samp = temp_shape[1];
 
               int32_t * raw_weight_index = extract_buffer <int32_t> (
-                  weight_index, "weight_index", 1, temp_shape, {n_det});
+                  weight_index, "weight_index", 1, temp_shape, {n_det}
+              );
 
 // Handle the case of either 2 or 3 dims
               auto winfo = weights.request();
@@ -126,28 +129,35 @@ void register_ops_scan_map(py::module & m, char const * name) {
               if (winfo.ndim == 2) {
                   nnz = 1;
                   raw_weights = extract_buffer <double> (
-                      weights, "weights", 2, temp_shape, {-1, n_samp});
+                      weights, "weights", 2, temp_shape, {-1, n_samp}
+                  );
               } else {
                   raw_weights = extract_buffer <double> (
-                      weights, "weights", 3, temp_shape, {-1, n_samp, -1});
+                      weights, "weights", 3, temp_shape, {-1, n_samp, -1}
+                  );
                   nnz = temp_shape[2];
               }
 
               int32_t * raw_data_index = extract_buffer <int32_t> (
-                  data_index, "data_index", 1, temp_shape, {n_det});
+                  data_index, "data_index", 1, temp_shape, {n_det}
+              );
               double * raw_det_data = extract_buffer <double> (
-                  det_data, "det_data", 2, temp_shape, {-1, n_samp});
+                  det_data, "det_data", 2, temp_shape, {-1, n_samp}
+              );
 
               Interval * raw_intervals = extract_buffer <Interval> (
-                  intervals, "intervals", 1, temp_shape, {-1});
+                  intervals, "intervals", 1, temp_shape, {-1}
+              );
               int64_t n_view = temp_shape[0];
 
               int64_t * raw_global2local = extract_buffer <int64_t> (
-                  global2local, "global2local", 1, temp_shape, {-1});
+                  global2local, "global2local", 1, temp_shape, {-1}
+              );
               int64_t n_global_submap = temp_shape[0];
 
               T * raw_mapdata = extract_buffer <T> (
-                  mapdata, "mapdata", 3, temp_shape, {-1, n_pix_submap, nnz});
+                  mapdata, "mapdata", 3, temp_shape, {-1, n_pix_submap, nnz}
+              );
               int64_t n_local_submap = temp_shape[0];
 
               if (offload) {
@@ -170,22 +180,31 @@ void register_ops_scan_map(py::module & m, char const * name) {
                       }
                   }
 
-                  # pragma omp target data map(to : raw_weight_index[0 : n_det], \
-                  raw_pixel_index[0 : n_det],                                    \
-                  raw_data_index[0 : n_det],                                     \
-                  raw_global2local[0 : n_global_submap],                         \
-                  n_view,                                                        \
-                  n_det,                                                         \
-                  n_samp,                                                        \
-                  max_interval_size,                                             \
-                  nnz,                                                           \
-                  n_pix_submap,                                                  \
-                  data_scale,                                                    \
-                  should_scale,                                                  \
-                  should_subtract,                                               \
-                  should_zero)
+                  # pragma omp target data map(          \
+                  to : raw_weight_index[0 : n_det],      \
+                  raw_pixel_index[0 : n_det],            \
+                  raw_data_index[0 : n_det],             \
+                  raw_global2local[0 : n_global_submap], \
+                  n_view,                                \
+                  n_det,                                 \
+                  n_samp,                                \
+                  max_interval_size,                     \
+                  nnz,                                   \
+                  n_pix_submap,                          \
+                  data_scale,                            \
+                  should_scale,                          \
+                  should_subtract,                       \
+                  should_zero                            \
+                  )
                   {
-                      # pragma omp target teams distribute parallel for collapse(3)
+                      # pragma omp target teams distribute parallel for collapse(3) \
+                      is_device_ptr(                                                \
+                      dev_pixels,                                                   \
+                      dev_weights,                                                  \
+                      dev_det_data,                                                 \
+                      dev_intervals,                                                \
+                      dev_mapdata                                                   \
+                      )
                       for (int64_t idet = 0; idet < n_det; idet++) {
                           for (int64_t iview = 0; iview < n_view; iview++) {
                               for (int64_t isamp = 0; isamp < max_interval_size;
@@ -216,7 +235,8 @@ void register_ops_scan_map(py::module & m, char const * name) {
                                       n_pix_submap,
                                       should_zero,
                                       should_subtract,
-                                      should_scale);
+                                      should_scale
+                                  );
                               }
                           }
                       }
@@ -230,7 +250,8 @@ void register_ops_scan_map(py::module & m, char const * name) {
                           for (
                               int64_t isamp = raw_intervals[iview].first;
                               isamp <= raw_intervals[iview].last;
-                              isamp++) {
+                              isamp++
+                          ) {
                               scan_map_inner <T> (
                                   raw_pixel_index,
                                   raw_weight_index,
@@ -248,14 +269,16 @@ void register_ops_scan_map(py::module & m, char const * name) {
                                   n_pix_submap,
                                   should_zero,
                                   should_subtract,
-                                  should_scale);
+                                  should_scale
+                              );
                           }
                       }
                   }
               }
 
               return;
-          });
+          }
+    );
     return;
 }
 
diff --git a/src/toast/_libtoast/ops_stokes_weights.cpp b/src/toast/_libtoast/ops_stokes_weights.cpp
index 75e3aed9a..b17d22230 100644
--- a/src/toast/_libtoast/ops_stokes_weights.cpp
+++ b/src/toast/_libtoast/ops_stokes_weights.cpp
@@ -33,23 +33,21 @@ void stokes_weights_qa_rotate(double const * q_in, double const * v_in,
     double z2 = -q_in[2] * q_in[2];
 
     v_out[0] = 2 * ((y2 + z2) * v_in[0] + (xy - zw) * v_in[1] +
-                    (yw + xz) * v_in[2]) +
-               v_in[0];
+                    (yw + xz) * v_in[2]) + v_in[0];
 
     v_out[1] = 2 * ((zw + xy) * v_in[0] + (x2 + z2) * v_in[1] +
-                    (yz - xw) * v_in[2]) +
-               v_in[1];
+                    (yz - xw) * v_in[2]) + v_in[1];
 
     v_out[2] = 2 * ((xz - yw) * v_in[0] + (xw + yz) * v_in[1] +
-                    (x2 + y2) * v_in[2]) +
-               v_in[2];
+                    (x2 + y2) * v_in[2]) + v_in[2];
 
     return;
 }
 
 void stokes_weights_alpha(
     double const * quats,
-    double * alpha) {
+    double * alpha
+) {
     const double xaxis[3] = {1.0, 0.0, 0.0};
     const double zaxis[3] = {0.0, 0.0, 1.0};
     double vd[3];
@@ -63,9 +61,10 @@ void stokes_weights_alpha(
     double vm_y = vd[2] * sin(ang_xy);
     double vm_z = -sqrt(1.0 - vd[2] * vd[2]);
 
-    double alpha_y =
-        (vd[0] * (vm_y * vo[2] - vm_z * vo[1]) - vd[1] * (vm_x * vo[2] - vm_z * vo[0]) +
-         vd[2] * (vm_x * vo[1] - vm_y * vo[0]));
+    double alpha_y = (
+        vd[0] * (vm_y * vo[2] - vm_z * vo[1]) - vd[1] * (vm_x * vo[2] - vm_z * vo[0]) +
+        vd[2] * (vm_x * vo[1] - vm_y * vo[0])
+    );
     double alpha_x = (vm_x * vo[0] + vm_y * vo[1] + vm_z * vo[2]);
 
     (*alpha) = atan2(alpha_y, alpha_x);
@@ -82,7 +81,8 @@ void stokes_weights_IQU_inner(
     int64_t isamp,
     int64_t n_samp,
     int64_t idet,
-    double U_sign) {
+    double U_sign
+) {
     double eta = (1.0 - epsilon[idet]) / (1.0 + epsilon[idet]);
     int32_t q_indx = quat_index[idet];
     int32_t w_indx = weight_index[idet];
@@ -114,7 +114,8 @@ void stokes_weights_IQU_inner_hwp(
     int64_t isamp,
     int64_t n_samp,
     int64_t idet,
-    double U_sign) {
+    double U_sign
+) {
     double eta = (1.0 - epsilon[idet]) / (1.0 + epsilon[idet]);
     int32_t q_indx = quat_index[idet];
     int32_t w_indx = weight_index[idet];
@@ -155,8 +156,8 @@ void init_ops_stokes_weights(py::module & m) {
             py::buffer gamma,
             double cal,
             bool IAU,
-            bool use_accel)
-        {
+            bool use_accel
+        ) {
             // NOTE:  Flags are not needed here, since the quaternions
             // have already had bad samples converted to null rotations.
 
@@ -232,20 +233,27 @@ void init_ops_stokes_weights(py::module & m) {
                     }
                 }
 
-                # pragma omp target data map(to : raw_weight_index[0 : n_det], \
-                raw_quat_index[0 : n_det],                                     \
-                raw_epsilon[0 : n_det],                                        \
-                raw_gamma[0 : n_det],                                          \
-                cal,                                                           \
-                U_sign,                                                        \
-                n_view,                                                        \
-                n_det,                                                         \
-                n_samp,                                                        \
-                max_interval_size)
+                # pragma omp target data map(     \
+                to : raw_weight_index[0 : n_det], \
+                raw_quat_index[0 : n_det],        \
+                raw_epsilon[0 : n_det],           \
+                raw_gamma[0 : n_det],             \
+                cal,                              \
+                U_sign,                           \
+                n_view,                           \
+                n_det,                            \
+                n_samp,                           \
+                max_interval_size                 \
+                )
                 {
                     if (!use_hwp) {
                         // No HWP
-                        # pragma omp target teams distribute parallel for collapse(3)
+                        # pragma omp target teams distribute parallel for collapse(3) \
+                        is_device_ptr(                                                \
+                        dev_weights,                                                  \
+                        dev_quats,                                                    \
+                        dev_intervals                                                 \
+                        )
                         for (int64_t idet = 0; idet < n_det; idet++) {
                             for (int64_t iview = 0; iview < n_view; iview++) {
                                 for (int64_t isamp = 0; isamp < max_interval_size;
@@ -276,7 +284,13 @@ void init_ops_stokes_weights(py::module & m) {
                         }
                     } else {
                         // We have a HWP
-                        # pragma omp target teams distribute parallel for collapse(3)
+                        # pragma omp target teams distribute parallel for collapse(3) \
+                        is_device_ptr(                                                \
+                        dev_weights,                                                  \
+                        dev_quats,                                                    \
+                        dev_hwp,                                                      \
+                        dev_intervals                                                 \
+                        )
                         for (int64_t idet = 0; idet < n_det; idet++) {
                             for (int64_t iview = 0; iview < n_view; iview++) {
                                 for (int64_t isamp = 0; isamp < max_interval_size;
@@ -375,8 +389,8 @@ void init_ops_stokes_weights(py::module & m) {
             py::buffer weights,
             py::buffer intervals,
             double cal,
-            bool use_accel)
-        {
+            bool use_accel
+        ) {
             // NOTE:  Flags are not needed here, since the quaternions
             // have already had bad samples converted to null rotations.
 
@@ -419,10 +433,19 @@ void init_ops_stokes_weights(py::module & m) {
                     }
                 }
 
-                # pragma \
-                omp target data map(to : raw_weight_index[0 : n_det], n_view, n_det, n_samp, cal)
+                # pragma omp target data map(     \
+                to : raw_weight_index[0 : n_det], \
+                n_view,                           \
+                n_det,                            \
+                n_samp,                           \
+                cal                               \
+                )
                 {
-                    # pragma omp target teams distribute parallel for collapse(3)
+                    # pragma omp target teams distribute parallel for collapse(3) \
+                    is_device_ptr(                                                \
+                    dev_weights,                                                  \
+                    dev_intervals                                                 \
+                    )
                     for (int64_t idet = 0; idet < n_det; idet++) {
                         for (int64_t iview = 0; iview < n_view; iview++) {
                             for (int64_t isamp = 0; isamp < max_interval_size; isamp++) {
diff --git a/src/toast/_libtoast/template_offset.cpp b/src/toast/_libtoast/template_offset.cpp
index fd7058e28..21bb00eed 100644
--- a/src/toast/_libtoast/template_offset.cpp
+++ b/src/toast/_libtoast/template_offset.cpp
@@ -21,8 +21,8 @@ void init_template_offset(py::module & m) {
             int32_t data_index,
             py::buffer det_data,
             py::buffer intervals,
-            bool use_accel)
-        {
+            bool use_accel
+        ) {
             auto & omgr = OmpManager::get();
             int dev = omgr.get_device();
             bool offload = (!omgr.device_is_host()) && use_accel;
@@ -73,14 +73,21 @@ void init_template_offset(py::module & m) {
                     }
                 }
 
-                # pragma omp target data map(to : n_view, \
-                n_samp,                                   \
-                data_index,                               \
-                step_length,                              \
-                amp_offset,                               \
-                amp_view_off[0 : n_view])
+                # pragma omp target data map( \
+                to : n_view,                  \
+                n_samp,                       \
+                data_index,                   \
+                step_length,                  \
+                amp_offset,                   \
+                amp_view_off[0 : n_view]      \
+                )
                 {
-                    # pragma omp target teams distribute parallel for collapse(2)
+                    # pragma omp target teams distribute parallel for collapse(2) \
+                    is_device_ptr(                                                \
+                    dev_amplitudes,                                               \
+                    dev_det_data,                                                 \
+                    dev_intervals                                                 \
+                    )
                     for (int64_t iview = 0; iview < n_view; iview++) {
                         for (int64_t isamp = 0; isamp < max_interval_size; isamp++) {
                             // Adjust for the actual start of the interval
@@ -137,8 +144,8 @@ void init_template_offset(py::module & m) {
             py::buffer n_amp_views,
             py::buffer amplitudes,
             py::buffer intervals,
-            bool use_accel)
-        {
+            bool use_accel
+        ) {
             auto & omgr = OmpManager::get();
             int dev = omgr.get_device();
             bool offload = (!omgr.device_is_host()) && use_accel;
@@ -200,20 +207,31 @@ void init_template_offset(py::module & m) {
                     }
                 }
 
-                # pragma omp target data map(to : n_view, \
-                n_samp,                                   \
-                data_index,                               \
-                flag_index,                               \
-                step_length,                              \
-                amp_offset,                               \
-                amp_view_off[0 : n_view],                 \
-                use_flags)
+                # pragma omp target data map( \
+                to : n_view,                  \
+                n_samp,                       \
+                data_index,                   \
+                flag_index,                   \
+                step_length,                  \
+                amp_offset,                   \
+                amp_view_off[0 : n_view],     \
+                use_flags                     \
+                )
                 {
                     // TODO the paralelism can likely be improved on this function
-                    # pragma omp target teams distribute collapse(2)
+                    # pragma omp target teams distribute collapse(2) \
+                    is_device_ptr(                                   \
+                    dev_amplitudes,                                  \
+                    dev_det_data,                                    \
+                    dev_det_flags,                                   \
+                    dev_intervals                                    \
+                    )
                     for (int64_t iview = 0; iview < n_view; iview++) {
-                        for (int64_t isamp = 0; isamp < max_interval_size;
-                             isamp += step_length) {
+                        for (
+                            int64_t isamp = 0;
+                            isamp < max_interval_size;
+                            isamp += step_length
+                        ) {
                             // Adjust for the actual start of the interval
                             int64_t adjusted_isamp = isamp + dev_intervals[iview].first;
 
@@ -224,9 +242,10 @@ void init_template_offset(py::module & m) {
                             }
 
                             // Insure we do not go out of the current interval
-                            int64_t max_step_length =
-                                std::min(step_length,
-                                         dev_intervals[iview].last - adjusted_isamp + 1);
+                            int64_t max_step_length = std::min(
+                                step_length,
+                                dev_intervals[iview].last - adjusted_isamp + 1
+                            );
 
                             // Reduce on a chunk of `step_length` samples.
                             double contrib = 0.0;
@@ -288,8 +307,8 @@ void init_template_offset(py::module & m) {
             py::buffer offset_var,
             py::buffer amplitudes_in,
             py::buffer amplitudes_out,
-            bool use_accel)
-        {
+            bool use_accel
+        ) {
             auto & omgr = OmpManager::get();
             int dev = omgr.get_device();
             bool offload = (!omgr.device_is_host()) && use_accel;
@@ -319,7 +338,12 @@ void init_template_offset(py::module & m) {
 
                 # pragma omp target data map(to : n_amp)
                 {
-                    # pragma omp target teams distribute parallel for
+                    # pragma omp target teams distribute parallel for \
+                    is_device_ptr(                                    \
+                    dev_amp_in,                                       \
+                    dev_amp_out,                                      \
+                    dev_offset_var                                    \
+                    )
                     for (int64_t iamp = 0; iamp < n_amp; iamp++) {
                         dev_amp_out[iamp] = dev_amp_in[iamp];
                         dev_amp_out[iamp] *= dev_offset_var[iamp];
diff --git a/wheels/install_deps_linux.sh b/wheels/install_deps_linux.sh
index e6a227754..b770eaa43 100755
--- a/wheels/install_deps_linux.sh
+++ b/wheels/install_deps_linux.sh
@@ -9,6 +9,26 @@
 
 set -e
 
+toolchain=$1
+PREFIX=$2
+static=$3
+
+# For testing locally
+# MAKEJ=8
+MAKEJ=2
+
+if [ "x${toolchain}" = "x" ]; then
+    toolchain="gcc"
+fi
+
+if [ "x${PREFIX}" = "x" ]; then
+    PREFIX=/usr/local
+fi
+
+if [ "x${static}" = "x" ]; then
+    static="yes"
+fi
+
 # Location of this script
 pushd $(dirname $0) >/dev/null 2>&1
 scriptdir=$(pwd)
@@ -17,23 +37,30 @@ echo "Wheel script directory = ${scriptdir}"
 
 # Build options
 
-CC=gcc
-CXX=g++
-FC=gfortran
-
-CFLAGS="-O3 -fPIC -pthread"
-FCFLAGS="-O3 -fPIC -pthread"
-CXXFLAGS="-O3 -fPIC -pthread -std=c++11"
-
-# For testing locally
-# MAKEJ=8
-MAKEJ=2
-
-PREFIX=/usr
-
-# Use yum to install OS packages
-
-yum update -y
+if [ "x${toolchain}" = "xgcc" ]; then
+    CC=gcc
+    CXX=g++
+    FC=gfortran
+
+    CFLAGS="-O3 -fPIC -pthread"
+    FCFLAGS="-O3 -fPIC -pthread"
+    CXXFLAGS="-O3 -fPIC -pthread -std=c++11"
+    FCLIBS="-lgfortran"
+else
+    if [ "x${toolchain}" = "xllvm" ]; then
+        CC=clang-17
+        CXX=clang++-17
+        FC=gfortran
+
+        CFLAGS="-O3 -fPIC -pthread"
+        FCFLAGS="-O3 -fPIC -pthread"
+        CXXFLAGS="-O3 -fPIC -pthread -std=c++11 -stdlib=libc++"
+        FCLIBS="-L/usr/lib/llvm-17/lib /usr/lib/x86_64-linux-gnu/libgfortran.so.5"
+    else
+        echo "Unsupported toolchain \"${toolchain}\""
+        exit 1
+    fi
+fi
 
 # Update pip
 pip install --upgrade pip
@@ -62,7 +89,7 @@ CC="${CC}" CFLAGS="${CFLAGS}" pip install -v "numpy<${numpy_ver}" -r "${scriptdi
 
 # Install Openblas
 
-openblas_version=0.3.21
+openblas_version=0.3.23
 openblas_dir=OpenBLAS-${openblas_version}
 openblas_pkg=${openblas_dir}.tar.gz
 
@@ -73,15 +100,20 @@ fi
 
 echo "Building OpenBLAS..."
 
+shr="NO_STATIC=1"
+if [ "${static}" = "yes" ]; then
+    shr="NO_SHARED=1"
+fi
+
 rm -rf ${openblas_dir}
 tar xzf ${openblas_pkg} \
     && pushd ${openblas_dir} >/dev/null 2>&1 \
-    && make USE_OPENMP=1 NO_SHARED=1 \
+    && make USE_OPENMP=1 ${shr} \
     MAKE_NB_JOBS=${MAKEJ} \
     CC="${CC}" FC="${FC}" DYNAMIC_ARCH=1 TARGET=GENERIC \
     COMMON_OPT="${CFLAGS}" FCOMMON_OPT="${FCFLAGS}" \
-    LDFLAGS="-fopenmp -lm" libs netlib \
-    && make NO_SHARED=1 DYNAMIC_ARCH=1 TARGET=GENERIC PREFIX="${PREFIX}" install \
+    EXTRALIB="-fopenmp -lm ${FCLIBS}" all \
+    && make ${shr} DYNAMIC_ARCH=1 TARGET=GENERIC PREFIX="${PREFIX}" install \
     && popd >/dev/null 2>&1
 
 # Install FFTW
@@ -98,15 +130,18 @@ fi
 
 echo "Building FFTW..."
 
+shr="--enable-shared --disable-static"
+if [ "${static}" = "yes" ]; then
+    shr="--enable-static --disable-shared"
+fi
+
 rm -rf ${fftw_dir}
 tar xzf ${fftw_pkg} \
     && pushd ${fftw_dir} >/dev/null 2>&1 \
     && CC="${CC}" CFLAGS="${CFLAGS}" \
     ./configure \
     --disable-fortran \
-    --enable-openmp \
-    --enable-static \
-    --disable-shared \
+    --enable-openmp ${shr} \
     --prefix="${PREFIX}" \
     && make -j ${MAKEJ} \
     && make install \
@@ -145,7 +180,7 @@ tar xzf ${aatm_pkg} \
 
 # Install libFLAC
 
-flac_version=1.4.2
+flac_version=1.4.3
 flac_dir=flac-${flac_version}
 flac_pkg=${flac_dir}.tar.gz
 
@@ -179,7 +214,7 @@ tar xzf ${flac_pkg} \
 
 # Install SuiteSparse
 
-ssparse_version=6.0.3
+ssparse_version=7.1.0
 ssparse_dir=SuiteSparse-${ssparse_version}
 ssparse_pkg=${ssparse_dir}.tar.gz
 
@@ -191,6 +226,11 @@ fi
 
 echo "Building SuiteSparse..."
 
+shr="-DNSTATIC:BOOL=ON -DBLA_STATIC:BOOL=OFF"
+if [ "${static}" = "yes" ]; then
+    shr="-DNSTATIC:BOOL=OFF -DBLA_STATIC:BOOL=ON"
+fi
+
 rm -rf ${ssparse_dir}
 tar xzf ${ssparse_pkg} \
     && pushd ${ssparse_dir} >/dev/null 2>&1 \
@@ -205,12 +245,10 @@ tar xzf ${ssparse_pkg} \
     -DCMAKE_Fortran_FLAGS=\"${FCFLAGS}\" \
     -DCMAKE_VERBOSE_MAKEFILE:BOOL=ON \
     -DCMAKE_INSTALL_PATH=\"${PREFIX}\" \
-    -DNSTATIC:BOOL=OFF \
     -DCMAKE_BUILD_TYPE=Release \
-    -DBLA_VENDOR=OpenBLAS \
-    -DBLA_STATIC:BOOL=ON \
-    -DBLAS_LIBRARIES=\"-L${PREFIX}/lib -lopenblas -lgomp -lm -lgfortran\" \
-    -DLAPACK_LIBRARIES=\"-L${PREFIX}/lib -lopenblas -lgomp -lm -lgfortran\" \
+    -DBLA_VENDOR=OpenBLAS ${shr} \
+    -DBLAS_LIBRARIES=\"-L${PREFIX}/lib -lopenblas -lm ${FCLIBS}\" \
+    -DLAPACK_LIBRARIES=\"-L${PREFIX}/lib -lopenblas -lm ${FCLIBS}\" \
     " make local \
     && make install \
     && cp ./lib/*.a "${PREFIX}/lib/" \
diff --git a/wheels/install_deps_osx.sh b/wheels/install_deps_osx.sh
index 866ca0279..4e47d30ff 100755
--- a/wheels/install_deps_osx.sh
+++ b/wheels/install_deps_osx.sh
@@ -219,7 +219,7 @@ tar xzf ${aatm_pkg} \
 
 # Install libFLAC
 
-flac_version=1.4.2
+flac_version=1.4.3
 flac_dir=flac-${flac_version}
 flac_pkg=${flac_dir}.tar.gz
 
diff --git a/wheels/suitesparse.patch b/wheels/suitesparse.patch
index 3ea600676..669f081a6 100644
--- a/wheels/suitesparse.patch
+++ b/wheels/suitesparse.patch
@@ -1,6 +1,6 @@
-diff -urN SuiteSparse-6.0.3_orig/Makefile SuiteSparse-6.0.3/Makefile
---- SuiteSparse-6.0.3_orig/Makefile	2022-12-24 19:37:26.000000000 -0800
-+++ SuiteSparse-6.0.3/Makefile	2023-01-17 09:08:51.599056620 -0800
+diff -urN SuiteSparse-7.1.0_orig/Makefile SuiteSparse-7.1.0/Makefile
+--- SuiteSparse-7.1.0_orig/Makefile	2023-06-29 19:06:35.000000000 -0700
++++ SuiteSparse-7.1.0/Makefile	2023-07-19 06:43:06.142861843 -0700
 @@ -39,8 +39,6 @@
  	( cd SuiteSparse_GPURuntime && $(MAKE) )
  	( cd GPUQREngine && $(MAKE) )
@@ -26,18 +26,9 @@ diff -urN SuiteSparse-6.0.3_orig/Makefile SuiteSparse-6.0.3/Makefile
 -	( cd GraphBLAS && $(MAKE) global )
 -	( cd SPEX && $(MAKE) global )
  
- # compile; "sudo make install" will install only in /usr/local
- # (or whatever your CMAKE_INSTALL_PREFIX is)
-@@ -107,8 +101,6 @@
- 	( cd SuiteSparse_GPURuntime && $(MAKE) both )
- 	( cd GPUQREngine && $(MAKE) both )
- 	( cd SPQR && $(MAKE) both )
--	( cd GraphBLAS && $(MAKE) both )
--	( cd SPEX && $(MAKE) both )
- 
  # install all packages.  Location depends on prior "make", "make global" etc
  install:
-@@ -128,8 +120,6 @@
+@@ -105,8 +99,6 @@
  	( cd SuiteSparse_GPURuntime && $(MAKE) install )
  	( cd GPUQREngine && $(MAKE) install )
  	( cd SPQR && $(MAKE) install )
@@ -46,7 +37,7 @@ diff -urN SuiteSparse-6.0.3_orig/Makefile SuiteSparse-6.0.3/Makefile
  
  # uninstall all packages
  uninstall:
-@@ -149,8 +139,6 @@
+@@ -126,8 +118,6 @@
  	( cd SuiteSparse_GPURuntime && $(MAKE) uninstall )
  	( cd GPUQREngine && $(MAKE) uninstall )
  	( cd SPQR && $(MAKE) uninstall )
@@ -55,16 +46,16 @@ diff -urN SuiteSparse-6.0.3_orig/Makefile SuiteSparse-6.0.3/Makefile
  
  # Remove all files not in the original distribution
  distclean: purge
-@@ -177,8 +165,6 @@
+@@ -154,8 +144,6 @@
  	- $(RM) MATLAB_Tools/*/*.mex* MATLAB_Tools/*/*/*.mex*
  	- $(RM) MATLAB_Tools/*/*.o    MATLAB_Tools/*/*/*.o
- 	- $(RM) -r include/* bin/* lib/* Example/build/*
+ 	- $(RM) -r Example/build/*
 -	- ( cd GraphBLAS && $(MAKE) purge )
 -	- ( cd SPEX && $(MAKE) purge )
+ 	- $(RM) -r include/* bin/* lib/*
  
  clean: purge
- 
-@@ -199,8 +185,6 @@
+@@ -177,12 +165,9 @@
  	- ( cd CXSparse && $(MAKE) demos )
  	- ( cd RBio && $(MAKE) demos )
  	- ( cd SPQR && $(MAKE) demos )
@@ -73,3 +64,36 @@ diff -urN SuiteSparse-6.0.3_orig/Makefile SuiteSparse-6.0.3/Makefile
  
  # Create the PDF documentation
  docs:
+-	( cd GraphBLAS && $(MAKE) docs )
+ 	( cd Mongoose  && $(MAKE) docs )
+ 	( cd AMD && $(MAKE) docs )
+ 	( cd CAMD && $(MAKE) docs )
+@@ -191,7 +176,6 @@
+ 	( cd UMFPACK && $(MAKE) docs )
+ 	( cd CHOLMOD && $(MAKE) docs )
+ 	( cd SPQR && $(MAKE) docs )
+-	( cd SPEX && $(MAKE) docs )
+ 
+ # statement coverage (Linux only); this requires a lot of time.
+ cov: local install
+@@ -201,13 +185,6 @@
+ 	( cd KLU && $(MAKE) cov )
+ 	( cd SPQR && $(MAKE) cov )
+ 	( cd UMFPACK && $(MAKE) cov )
+-	( cd SPEX && $(MAKE) cov )
+-
+-gbmatlab:
+-	( cd GraphBLAS/GraphBLAS && $(MAKE) )
+-
+-gblocal:
+-	( cd GraphBLAS/GraphBLAS && $(MAKE) local && $(MAKE) install )
+ 
+ debug:
+ 	( cd SuiteSparse_config && $(MAKE) debug )
+@@ -227,6 +204,4 @@
+ 	( cd SuiteSparse_GPURuntime && $(MAKE) )
+ 	( cd GPUQREngine && $(MAKE) )
+ 	( cd SPQR && $(MAKE) debug )
+-	( cd GraphBLAS && $(MAKE) debug )
+-	( cd SPEX && $(MAKE) debug )
+ 


Test patch:
diff --git a/wheels/test_local_linux.sh b/wheels/test_local_linux.sh
new file mode 100755
index 000000000..22b4a9264
--- /dev/null
+++ b/wheels/test_local_linux.sh
@@ -0,0 +1,92 @@
+#!/bin/bash
+
+# This script is used to build toast and dependencies and install them
+# into the currently active virtualenv.
+
+set -e
+
+# Location of the source tree
+pushd $(dirname $0) >/dev/null 2>&1
+topdir=$(dirname $(pwd))
+popd >/dev/null 2>&1
+
+venv_path=$1
+if [ "x${venv_path}" = "x" ]; then
+    echo "Usage:  $0 <virtualenv path>"
+    echo "  If the path to the virtualenv exists, it will be activated."
+    echo "  Otherwise it will be created."
+    exit 1
+fi
+
+toolchain=$2
+if [ "x${toolchain}" = "x" ]; then
+    toolchain="gcc"
+fi
+
+# Deactivate any current venv
+if [ "x$(type -t deactivate)" != "x" ]; then
+    deactivate
+fi
+
+echo "Using compiler '${toolchain}'"
+
+if [ -d ${venv_path} ]; then
+    echo "Virtualenv \"${venv_path}\" already exists, activating"
+else
+    echo "Creating virtualenv \"${venv_path}\""
+    eval "python3 -m venv \"${venv_path}\""
+fi
+source "${venv_path}/bin/activate"
+venv_py_ver=$(python3 --version | awk '{print $2}')
+
+# Install our dependencies
+eval "${topdir}/wheels/install_deps_linux.sh" "${toolchain}" "${venv_path}" "no"
+
+exit 0
+
+# Look for our dependencies in the virtualenv
+export LD_LIBRARY_PATH="${venv_path}/lib"
+export CPATH="${venv_path}/include"
+
+if [ "x${toolchain}" = "xgcc" ]; then
+    CC=gcc
+    CXX=g++
+    CFLAGS="-O3 -fPIC -pthread"
+    CXXFLAGS="-O3 -fPIC -pthread -std=c++11"
+    FCLIBS="-lgfortran"
+else
+    if [ "x${toolchain}" = "xllvm" ]; then
+        CC=clang-17
+        CXX=clang++-17
+        CFLAGS="-O3 -fPIC -pthread"
+        CXXFLAGS="-O3 -fPIC -pthread -std=c++11 -stdlib=libc++"
+        FCLIBS="-lgfortran"
+    else
+        echo "Unsupported toolchain \"${toolchain}\""
+        exit 1
+    fi
+fi
+
+# Set up toast build options
+export TOAST_BUILD_CMAKE_C_COMPILER="${CC}"
+export TOAST_BUILD_CMAKE_CXX_COMPILER="${CXX}"
+export TOAST_BUILD_CMAKE_C_FLAGS="${CFLAGS} -I${venv_path}/include"
+export TOAST_BUILD_CMAKE_CXX_FLAGS="${CXXFLAGS} -I${venv_path}/include"
+export TOAST_BUILD_DISABLE_OPENMP_TARGET=ON
+export TOAST_BUILD_BLAS_LIBRARIES="-L${venv_path}/lib -lopenblas -fopenmp -lm ${FCLIBS}"
+export TOAST_BUILD_LAPACK_LIBRARIES="-L${venv_path}/lib -lopenblas -fopenmp -lm ${FCLIBS}"
+export TOAST_BUILD_CMAKE_VERBOSE_MAKEFILE=ON
+export TOAST_BUILD_AATM_ROOT="${venv_path}"
+export TOAST_BUILD_FFTW_ROOT="${venv_path}"
+export TOAST_BUILD_SUITESPARSE_INCLUDE_DIR_HINTS="${venv_path}/include"
+export TOAST_BUILD_SUITESPARSE_LIBRARY_DIR_HINTS="${venv_path}/lib"
+export TOAST_BUILD_CMAKE_LIBRARY_PATH="${venv_path}/lib"
+# export TOAST_BUILD_TOAST_STATIC_DEPS=ON
+
+# Install it
+pushd "${topdir}" 2>&1 >/dev/null
+python3 -m pip install -vvv .
+popd 2>&1 >/dev/null
+
+# Run tests
+python3 -c 'import toast.tests; toast.tests.run()'