Repository: lanl/ports-of-call
Pull request #83: Add PORTABLE_FENCE
Author: login: jhp-lanl
Created: 2025-03-04T22:35:13Z | Merged: 2025-03-06T04:56:02Z

<!--Provide a general summary of your changes in the title above, for
example "Improve interpToDB routines.".  Please avoid
non-descriptive titles such as "Addresses issue #8576".-->

## PR Summary

Adds a `PORTABLE_FENCE` macro

<!--Please provide at least 1-2 sentences describing the pull request in
detail.  Why is this change required?  What problem does it solve?-->

<!--If it fixes an open issue, please link to the issue here.-->

## PR Checklist

<!-- Note that some of these check boxes may not apply to all pull requests -->

- [x] Any changes to code are appropriately documented.
- [x] Code is formatted.
- [x] Install test passes.
- [x] Docs build.
- [ ] If preparing for a new release, update the version in cmake.



Diff:
diff --git a/.github/workflows/formatting.yml b/.github/workflows/formatting.yml
index a3416ee5..23a23635 100644
--- a/.github/workflows/formatting.yml
+++ b/.github/workflows/formatting.yml
@@ -9,7 +9,7 @@ on:
 jobs:
     formatting:
       name: Check Formatting
-      runs-on: ubuntu-latest
+      runs-on: ubuntu-22.04
 
       steps:      
         - name: Checkout code
@@ -18,7 +18,7 @@ jobs:
           run: export DEBIAN_FRONTEND=noninteractive
         - name: install dependencies
           run: |
-            sudo apt-get install -y --force-yes -qq git clang-format
+            sudo apt-get install -y --force-yes -qq git clang-format-12
         - name: check formatting
           continue-on-error: true
-          run: find . -regex '.*\.\(cpp\|hpp\)' | xargs clang-format -style=file -i && git diff --exit-code --ignore-submodules
+          run: find . -regex '.*\.\(cpp\|hpp\)' | xargs clang-format-12 -style=file -i && git diff --exit-code --ignore-submodules
diff --git a/doc/sphinx/src/using.rst b/doc/sphinx/src/using.rst
index 82ffafd4..a8c316b3 100644
--- a/doc/sphinx/src/using.rst
+++ b/doc/sphinx/src/using.rst
@@ -22,6 +22,7 @@ To include Ports of Call in your project, simply include the directory
 6. ``_WITH_CUDA_``: Defined when Cuda is enabled
 7. ``Real``: a typedef to double (default) or float (if you define ``SINGLE_PRECISION_ENABLED``)
 8. ``PORTABLE_MALLOC()``, ``PORTABLE_FREE()``: A wrapper for kokkos_malloc or cudaMalloc, or raw malloc and equivalent free.
+9. ``PORTABLE_FENCE()``: A wrapper for ``kokkos::fence`` or ``cudaDeviceSynchronize()``
 
 At compile time, you define
 ``PORTABILITY_STRATEGY_{KOKKOS,CUDA,NONE}`` (if you don't define it,
@@ -71,7 +72,9 @@ limited. The syntax is:
     Function function, T &reduced) {
 
 where ``Function`` now takes as many indices are required and
-``reduced`` as arguments.
+``reduced`` as arguments. Note that a ``portableReduce()`` is blocking (i.e. a
+device synchronization step is performed) while `portableFor()` may not be,
+possibly requiring a ``PORTABLE_FENCE`` to avoid any race conditions.
 
 Also provided are host to device and device to host memory transfers of the form:
 
diff --git a/ports-of-call/math_utils.hpp b/ports-of-call/math_utils.hpp
index 5202e156..2b6070c9 100644
--- a/ports-of-call/math_utils.hpp
+++ b/ports-of-call/math_utils.hpp
@@ -31,8 +31,7 @@ template <typename BaseT, typename ExponentT,
           typename std::enable_if<std::is_arithmetic_v<std::decay_t<BaseT>> &&
                                   std::is_integral_v<std::decay_t<ExponentT>>>::type * =
               nullptr>
-PORTABLE_FORCEINLINE_FUNCTION constexpr auto power(BaseT base,
-                                                                     ExponentT exponent) {
+PORTABLE_FORCEINLINE_FUNCTION constexpr auto power(BaseT base, ExponentT exponent) {
   using PowT = decltype(std::pow(base, exponent));
   if (!Robust::check_nonnegative(exponent) || exponent > ExponentT{100}) {
     return std::pow(base, exponent);
@@ -52,8 +51,8 @@ template <typename BaseT, typename ExponentT,
           typename std::enable_if<std::is_arithmetic_v<std::decay_t<BaseT>> &&
                                   std::is_floating_point_v<std::decay_t<ExponentT>>>::type
               * = nullptr>
-PORTABLE_FORCEINLINE_FUNCTION constexpr auto
-power(BaseT const &base, ExponentT const &exponent) {
+PORTABLE_FORCEINLINE_FUNCTION constexpr auto power(BaseT const &base,
+                                                   ExponentT const &exponent) {
   if (!Robust::check_nonnegative(base)) {
     return std::pow(base, exponent);
   }
@@ -66,8 +65,8 @@ template <typename BaseT, typename ExponentT,
           typename std::enable_if<not std::is_arithmetic_v<std::decay_t<BaseT>> ||
                                   not std::is_arithmetic_v<std::decay_t<ExponentT>>>::type
               * = nullptr>
-PORTABLE_FORCEINLINE_FUNCTION constexpr auto
-power(BaseT const &base, ExponentT const &exponent) {
+PORTABLE_FORCEINLINE_FUNCTION constexpr auto power(BaseT const &base,
+                                                   ExponentT const &exponent) {
   return std::pow(base, exponent);
 }
 
diff --git a/ports-of-call/portability.hpp b/ports-of-call/portability.hpp
index dd34ee80..af896db3 100644
--- a/ports-of-call/portability.hpp
+++ b/ports-of-call/portability.hpp
@@ -53,6 +53,7 @@
 #define PORTABLE_MALLOC(size) Kokkos::kokkos_malloc<>(size)
 #define PORTABLE_FREE(ptr) Kokkos::kokkos_free<>(ptr)
 // Do we want to include additional terms here (for memory spaces, etc.)?
+#define PORTABLE_FENCE(...) Kokkos::fence(__VA_ARGS__)
 #else
 #ifdef PORTABILITY_STRATEGY_CUDA
 // currently error out on cuda since its not implemented
@@ -69,6 +70,7 @@ void *PORTABLE_MALLOC(size_t size) {
   return devPtr;
 }
 void PORTABLE_FREE(void *ptr) { cudaError_t e = cudaFree(ptr); }
+#define PORTABLE_FENCE(...) cudaDeviceSynchronize()
 #define _WITH_CUDA_
 // It is worth noting here that we will not define
 // _WITH_CUDA_ when we are doing KOKKOS (even with the
@@ -80,6 +82,7 @@ void PORTABLE_FREE(void *ptr) { cudaError_t e = cudaFree(ptr); }
 #define PORTABLE_LAMBDA [=]
 #define PORTABLE_MALLOC(size) malloc(size)
 #define PORTABLE_FREE(ptr) free(ptr)
+#define PORTABLE_FENCE(...)
 #endif
 #endif
 
diff --git a/test/test_portability.cpp b/test/test_portability.cpp
index 04d4dad0..e854c274 100644
--- a/test/test_portability.cpp
+++ b/test/test_portability.cpp
@@ -132,3 +132,59 @@ TEST_CASE("portableCopy works with all portability strategies", "[portableCopy]"
   // free device memory
   PORTABLE_FREE(a);
 }
+
+PORTABLE_FORCEINLINE_FUNCTION void expensive_operation(int iterations) {
+  double sum = 0.0;
+  for (int i = 0; i < iterations; i++) {
+    sum += 1.0 / (i + 1.0); // Expensive floating-point operation
+  }
+  // Prevent compiler optimization
+  if (sum < 0) {
+    printf("Artificial delay: %f\n", sum);
+  }
+}
+
+TEST_CASE("PORTABLE_FENCE properly synchronizes execution after a portableFor",
+          "[portableFor][PORTABLE_FENCE]") {
+
+  // Create host array
+  constexpr int N = 10000;
+  constexpr Real init_val = 2.0;
+  constexpr Real mult_factor = 50.;
+  std::vector<Real> h_data(N); // Zero initialized
+  for (size_t i = 0; i < N; i++) {
+    h_data[i] = init_val;
+  }
+
+  // Copy to device
+  constexpr const size_t bytes = N * sizeof(Real);
+  Real *d_data = (Real *)PORTABLE_MALLOC(bytes);
+  portableCopyToDevice(d_data, h_data.data(), bytes);
+
+  // Use portableFor since it doesn't have an implicit fence
+  portableFor(
+      "Expensive loop to test PORTABLE_FENCE", 0, N, PORTABLE_LAMBDA(const int i) {
+        expensive_operation(1e6);
+        d_data[i] *= mult_factor;
+      });
+
+  // Fence before reduction
+  PORTABLE_FENCE("Fence after expensive operation");
+
+  // JHP: it's not completely clear to us whether this test would fail
+  // without the fence. Are device loops inherently sequential?
+  int n_wrong = 0;
+  portableReduce(
+      "Check PORTABLE_FENCE", 0, N,
+      PORTABLE_LAMBDA(const int &i, int &n_wrong) {
+        if (d_data[i] != init_val * mult_factor) {
+          n_wrong += 1;
+        }
+      },
+      n_wrong);
+
+  REQUIRE(n_wrong == 0);
+
+  // free device memory
+  PORTABLE_FREE(d_data);
+}


Reviews:
user: login: jhp-lanl
state: COMMENTED
author_association: COLLABORATOR
submitted_at: 2025-03-05T02:44:05Z
user: login: jhp-lanl
state: COMMENTED
author_association: COLLABORATOR
submitted_at: 2025-03-05T02:44:46Z
user: login: BrendanKKrueger
state: COMMENTED
author_association: COLLABORATOR
submitted_at: 2025-03-05T15:36:25Z
user: login: Yurlungur
state: COMMENTED
author_association: COLLABORATOR
submitted_at: 2025-03-05T15:59:37Z
user: login: Yurlungur
state: COMMENTED
author_association: COLLABORATOR
submitted_at: 2025-03-05T16:02:31Z
user: login: Yurlungur
body: Some nitpicks below. Formatting should pass now.
state: APPROVED
author_association: COLLABORATOR
submitted_at: 2025-03-05T16:11:38Z
user: login: jhp-lanl
state: COMMENTED
author_association: COLLABORATOR
submitted_at: 2025-03-05T23:42:07Z
user: login: jhp-lanl
state: COMMENTED
author_association: COLLABORATOR
submitted_at: 2025-03-05T23:50:16Z
user: login: jhp-lanl
state: COMMENTED
author_association: COLLABORATOR
submitted_at: 2025-03-05T23:50:33Z
user: login: jhp-lanl
state: COMMENTED
author_association: COLLABORATOR
submitted_at: 2025-03-06T04:52:24Z

Review comments:
diff_hunk: @@ -31,8 +31,7 @@ template <typename BaseT, typename ExponentT,
           typename std::enable_if<std::is_arithmetic_v<std::decay_t<BaseT>> &&
                                   std::is_integral_v<std::decay_t<ExponentT>>>::type * =
               nullptr>
-PORTABLE_FORCEINLINE_FUNCTION constexpr auto power(BaseT base,
-                                                                     ExponentT exponent) {
+PORTABLE_FORCEINLINE_FUNCTION constexpr auto power(BaseT base, ExponentT exponent) {
path: ports-of-call/math_utils.hpp
user: login: jhp-lanl
body: These changes were apparently because clang format wasn't run (or maybe a different version?).
created_at: 2025-03-05T02:44:05Z
author_association: COLLABORATOR
diff_hunk: @@ -53,6 +53,7 @@
 #define PORTABLE_MALLOC(size) Kokkos::kokkos_malloc<>(size)
 #define PORTABLE_FREE(ptr) Kokkos::kokkos_free<>(ptr)
 // Do we want to include additional terms here (for memory spaces, etc.)?
+#define PORTABLE_FENCE(name) Kokkos::fence(name)
path: ports-of-call/portability.hpp
user: login: jhp-lanl
body: Do I have access to Kokkos execution spaces? If so, it would make more sense to fence the device execution space by itself.
created_at: 2025-03-05T02:44:46Z
author_association: COLLABORATOR
diff_hunk: @@ -31,8 +31,7 @@ template <typename BaseT, typename ExponentT,
           typename std::enable_if<std::is_arithmetic_v<std::decay_t<BaseT>> &&
                                   std::is_integral_v<std::decay_t<ExponentT>>>::type * =
               nullptr>
-PORTABLE_FORCEINLINE_FUNCTION constexpr auto power(BaseT base,
-                                                                     ExponentT exponent) {
+PORTABLE_FORCEINLINE_FUNCTION constexpr auto power(BaseT base, ExponentT exponent) {
path: ports-of-call/math_utils.hpp
user: login: BrendanKKrueger
body: I believe it -- I've had trouble getting all the clang-format stuff to agree on various places I was working on Ports-of-Call.  By the way, the format job is still failing.  I'm not sure why it says it passes, but if you open it up and look at the output you'll see that there are failures.
created_at: 2025-03-05T15:36:25Z
author_association: COLLABORATOR
diff_hunk: @@ -31,8 +31,7 @@ template <typename BaseT, typename ExponentT,
           typename std::enable_if<std::is_arithmetic_v<std::decay_t<BaseT>> &&
                                   std::is_integral_v<std::decay_t<ExponentT>>>::type * =
               nullptr>
-PORTABLE_FORCEINLINE_FUNCTION constexpr auto power(BaseT base,
-                                                                     ExponentT exponent) {
+PORTABLE_FORCEINLINE_FUNCTION constexpr auto power(BaseT base, ExponentT exponent) {
path: ports-of-call/math_utils.hpp
user: login: Yurlungur
body: The formatting is due to the CI for ports-of-call apparently not being pinned to a specific clang-format version. I pinned it. That said, at some point we should update to a newer version of clang format.
created_at: 2025-03-05T15:59:36Z
author_association: COLLABORATOR
diff_hunk: @@ -53,6 +53,7 @@
 #define PORTABLE_MALLOC(size) Kokkos::kokkos_malloc<>(size)
 #define PORTABLE_FREE(ptr) Kokkos::kokkos_free<>(ptr)
 // Do we want to include additional terms here (for memory spaces, etc.)?
+#define PORTABLE_FENCE(name) Kokkos::fence(name)
path: ports-of-call/portability.hpp
user: login: Yurlungur
body: We don't... I think adding execution spaces generically unfortunately moves us more towards mimicking additional kokkos functionality. Let's leave this as is for now but add an issue?
created_at: 2025-03-05T16:02:31Z
author_association: COLLABORATOR
diff_hunk: @@ -22,6 +22,7 @@ To include Ports of Call in your project, simply include the directory
 6. ``_WITH_CUDA_``: Defined when Cuda is enabled
 7. ``Real``: a typedef to double (default) or float (if you define ``SINGLE_PRECISION_ENABLED``)
 8. ``PORTABLE_MALLOC()``, ``PORTABLE_FREE()``: A wrapper for kokkos_malloc or cudaMalloc, or raw malloc and equivalent free.
+9. ``PORTABLE_FENCE()``: A wrapper for kokkos::fence or cudaDeviceSynchronize()
path: doc/sphinx/src/using.rst
user: login: Yurlungur
body: ```suggestion
9. ``PORTABLE_FENCE()``: A wrapper for ``kokkos::fence`` or ``cudaDeviceSynchronize()``
```
created_at: 2025-03-05T16:03:14Z
author_association: COLLABORATOR
diff_hunk: @@ -53,6 +53,7 @@
 #define PORTABLE_MALLOC(size) Kokkos::kokkos_malloc<>(size)
 #define PORTABLE_FREE(ptr) Kokkos::kokkos_free<>(ptr)
 // Do we want to include additional terms here (for memory spaces, etc.)?
+#define PORTABLE_FENCE(name) Kokkos::fence(name)
path: ports-of-call/portability.hpp
user: login: Yurlungur
body: So you don't always need a name:
```suggestion
#define PORTABLE_FENCE(...) Kokkos::fence(__VA_ARGS__)
```
created_at: 2025-03-05T16:04:38Z
author_association: COLLABORATOR
diff_hunk: @@ -69,6 +70,7 @@ void *PORTABLE_MALLOC(size_t size) {
   return devPtr;
 }
 void PORTABLE_FREE(void *ptr) { cudaError_t e = cudaFree(ptr); }
+#define PORTABLE_FENCE(name) cudaDeviceSynchronize()
path: ports-of-call/portability.hpp
user: login: Yurlungur
body: ```suggestion
#define PORTABLE_FENCE(...) cudaDeviceSynchronize()
```
created_at: 2025-03-05T16:05:02Z
author_association: COLLABORATOR
diff_hunk: @@ -80,6 +82,7 @@ void PORTABLE_FREE(void *ptr) { cudaError_t e = cudaFree(ptr); }
 #define PORTABLE_LAMBDA [=]
 #define PORTABLE_MALLOC(size) malloc(size)
 #define PORTABLE_FREE(ptr) free(ptr)
+#define PORTABLE_FENCE(name)
path: ports-of-call/portability.hpp
user: login: Yurlungur
body: ```suggestion
#define PORTABLE_FENCE(...)
```
created_at: 2025-03-05T16:05:23Z
author_association: COLLABORATOR
diff_hunk: @@ -132,3 +132,58 @@ TEST_CASE("portableCopy works with all portability strategies", "[portableCopy]"
   // free device memory
   PORTABLE_FREE(a);
 }
+
+PORTABLE_FORCEINLINE_FUNCTION void expensive_operation(int iterations) {
+  double sum = 0.0;
+  for (int i = 0; i < iterations; i++) {
+    sum += 1.0 / (i + 1.0); // Expensive floating-point operation
+  }
+  // Prevent compiler optimization
+  if (sum < 0) {
+    printf("Artificial delay: %f\n", sum);
+  }
+}
+
+TEST_CASE("PORTABLE_FENCE properly synchronizes execution after a portableFor",
+          "[portableFor][PORTABLE_FENCE]") {
+
+  // Create host array
+  constexpr int N = 100;
+  constexpr Real init_val = 2.0;
+  constexpr Real mult_factor = 50.;
+  std::vector<Real> h_data(N); // Zero initialized
+  for (size_t i = 0; i < N; i++) {
+    h_data[i] = init_val;
+  }
+
+  // Copy to device
+  constexpr const size_t bytes = N * sizeof(Real);
+  Real *d_data = (Real *)PORTABLE_MALLOC(bytes);
+  portableCopyToDevice(d_data, h_data.data(), bytes);
+
+  // Use portableFor since it doesn't have an implicit fence
+  portableFor(
+      "Expensive loop to test PORTABLE_FENCE", 0, N, PORTABLE_LAMBDA(const int i) {
+        expensive_operation(1e6);
+        d_data[i] *= mult_factor;
+      });
+
+  // Fence before reduction
+  PORTABLE_FENCE("Fence after expensive operation");
+
+  // Use a device operation that would fail if the fence wasn't in place
+  int n_wrong = 0;
+  portableReduce(
+      "Check PORTABLE_FENCE", 0, N,
+      PORTABLE_LAMBDA(const int &i, int &n_wrong) {
+        if (d_data[i] != init_val * mult_factor) {
+          n_wrong += 1;
+        }
+      },
+      n_wrong);
+
+  REQUIRE(n_wrong == 0);
path: test/test_portability.cpp
user: login: Yurlungur
body: I can't think of anything better, but I'm actually not sure this would necessarily fail if the fence wasn't in place... I *think* multiple `Kokkos::parallel_for`s in the same execution space somehow execute one-after-the-other. I have never fully understood the rules there.
created_at: 2025-03-05T16:11:23Z
author_association: COLLABORATOR
diff_hunk: @@ -53,6 +53,7 @@
 #define PORTABLE_MALLOC(size) Kokkos::kokkos_malloc<>(size)
 #define PORTABLE_FREE(ptr) Kokkos::kokkos_free<>(ptr)
 // Do we want to include additional terms here (for memory spaces, etc.)?
+#define PORTABLE_FENCE(name) Kokkos::fence(name)
path: ports-of-call/portability.hpp
user: login: jhp-lanl
body: 🤷 I'm not sure it even needs an issue since, as you said, that's going a bit beyond what we want to do.
created_at: 2025-03-05T23:42:07Z
author_association: COLLABORATOR
diff_hunk: @@ -132,3 +132,58 @@ TEST_CASE("portableCopy works with all portability strategies", "[portableCopy]"
   // free device memory
   PORTABLE_FREE(a);
 }
+
+PORTABLE_FORCEINLINE_FUNCTION void expensive_operation(int iterations) {
+  double sum = 0.0;
+  for (int i = 0; i < iterations; i++) {
+    sum += 1.0 / (i + 1.0); // Expensive floating-point operation
+  }
+  // Prevent compiler optimization
+  if (sum < 0) {
+    printf("Artificial delay: %f\n", sum);
+  }
+}
+
+TEST_CASE("PORTABLE_FENCE properly synchronizes execution after a portableFor",
+          "[portableFor][PORTABLE_FENCE]") {
+
+  // Create host array
+  constexpr int N = 100;
+  constexpr Real init_val = 2.0;
+  constexpr Real mult_factor = 50.;
+  std::vector<Real> h_data(N); // Zero initialized
+  for (size_t i = 0; i < N; i++) {
+    h_data[i] = init_val;
+  }
+
+  // Copy to device
+  constexpr const size_t bytes = N * sizeof(Real);
+  Real *d_data = (Real *)PORTABLE_MALLOC(bytes);
+  portableCopyToDevice(d_data, h_data.data(), bytes);
+
+  // Use portableFor since it doesn't have an implicit fence
+  portableFor(
+      "Expensive loop to test PORTABLE_FENCE", 0, N, PORTABLE_LAMBDA(const int i) {
+        expensive_operation(1e6);
+        d_data[i] *= mult_factor;
+      });
+
+  // Fence before reduction
+  PORTABLE_FENCE("Fence after expensive operation");
+
+  // Use a device operation that would fail if the fence wasn't in place
+  int n_wrong = 0;
+  portableReduce(
+      "Check PORTABLE_FENCE", 0, N,
+      PORTABLE_LAMBDA(const int &i, int &n_wrong) {
+        if (d_data[i] != init_val * mult_factor) {
+          n_wrong += 1;
+        }
+      },
+      n_wrong);
+
+  REQUIRE(n_wrong == 0);
path: test/test_portability.cpp
user: login: jhp-lanl
body: Yeaaaaaah I'm not sure. All I could discover was that there is an implicit fence at the end of a `parallel_reduce`, but that there _shouldn't_ be one at the end of `parallel_for`. I don't know what that means for what I've written though.
created_at: 2025-03-05T23:50:16Z
author_association: COLLABORATOR
diff_hunk: @@ -132,3 +132,58 @@ TEST_CASE("portableCopy works with all portability strategies", "[portableCopy]"
   // free device memory
   PORTABLE_FREE(a);
 }
+
+PORTABLE_FORCEINLINE_FUNCTION void expensive_operation(int iterations) {
+  double sum = 0.0;
+  for (int i = 0; i < iterations; i++) {
+    sum += 1.0 / (i + 1.0); // Expensive floating-point operation
+  }
+  // Prevent compiler optimization
+  if (sum < 0) {
+    printf("Artificial delay: %f\n", sum);
+  }
+}
+
+TEST_CASE("PORTABLE_FENCE properly synchronizes execution after a portableFor",
+          "[portableFor][PORTABLE_FENCE]") {
+
+  // Create host array
+  constexpr int N = 100;
+  constexpr Real init_val = 2.0;
+  constexpr Real mult_factor = 50.;
+  std::vector<Real> h_data(N); // Zero initialized
+  for (size_t i = 0; i < N; i++) {
+    h_data[i] = init_val;
+  }
+
+  // Copy to device
+  constexpr const size_t bytes = N * sizeof(Real);
+  Real *d_data = (Real *)PORTABLE_MALLOC(bytes);
+  portableCopyToDevice(d_data, h_data.data(), bytes);
+
+  // Use portableFor since it doesn't have an implicit fence
+  portableFor(
+      "Expensive loop to test PORTABLE_FENCE", 0, N, PORTABLE_LAMBDA(const int i) {
+        expensive_operation(1e6);
+        d_data[i] *= mult_factor;
+      });
+
+  // Fence before reduction
+  PORTABLE_FENCE("Fence after expensive operation");
+
+  // Use a device operation that would fail if the fence wasn't in place
+  int n_wrong = 0;
+  portableReduce(
+      "Check PORTABLE_FENCE", 0, N,
+      PORTABLE_LAMBDA(const int &i, int &n_wrong) {
+        if (d_data[i] != init_val * mult_factor) {
+          n_wrong += 1;
+        }
+      },
+      n_wrong);
+
+  REQUIRE(n_wrong == 0);
path: test/test_portability.cpp
user: login: jhp-lanl
body: I can update the comment to be less confident though
created_at: 2025-03-05T23:50:33Z
author_association: COLLABORATOR
diff_hunk: @@ -132,3 +132,58 @@ TEST_CASE("portableCopy works with all portability strategies", "[portableCopy]"
   // free device memory
   PORTABLE_FREE(a);
 }
+
+PORTABLE_FORCEINLINE_FUNCTION void expensive_operation(int iterations) {
+  double sum = 0.0;
+  for (int i = 0; i < iterations; i++) {
+    sum += 1.0 / (i + 1.0); // Expensive floating-point operation
+  }
+  // Prevent compiler optimization
+  if (sum < 0) {
+    printf("Artificial delay: %f\n", sum);
+  }
+}
+
+TEST_CASE("PORTABLE_FENCE properly synchronizes execution after a portableFor",
+          "[portableFor][PORTABLE_FENCE]") {
+
+  // Create host array
+  constexpr int N = 100;
+  constexpr Real init_val = 2.0;
+  constexpr Real mult_factor = 50.;
+  std::vector<Real> h_data(N); // Zero initialized
+  for (size_t i = 0; i < N; i++) {
+    h_data[i] = init_val;
+  }
+
+  // Copy to device
+  constexpr const size_t bytes = N * sizeof(Real);
+  Real *d_data = (Real *)PORTABLE_MALLOC(bytes);
+  portableCopyToDevice(d_data, h_data.data(), bytes);
+
+  // Use portableFor since it doesn't have an implicit fence
+  portableFor(
+      "Expensive loop to test PORTABLE_FENCE", 0, N, PORTABLE_LAMBDA(const int i) {
+        expensive_operation(1e6);
+        d_data[i] *= mult_factor;
+      });
+
+  // Fence before reduction
+  PORTABLE_FENCE("Fence after expensive operation");
+
+  // Use a device operation that would fail if the fence wasn't in place
+  int n_wrong = 0;
+  portableReduce(
+      "Check PORTABLE_FENCE", 0, N,
+      PORTABLE_LAMBDA(const int &i, int &n_wrong) {
+        if (d_data[i] != init_val * mult_factor) {
+          n_wrong += 1;
+        }
+      },
+      n_wrong);
+
+  REQUIRE(n_wrong == 0);
path: test/test_portability.cpp
user: login: jhp-lanl
body: Changed. I also increased the number of array items in case the first loop can continue on that task while the reduction goes in. The fence will ensure the answer doesn't change, but if somebody wants to test this, they can remove the fence and see what happens.
created_at: 2025-03-06T04:52:24Z
author_association: COLLABORATOR

PR comments:
user: login: jhp-lanl
created_at: 2025-03-04T22:48:47Z
body: I don't think the `Check Formatting` step is working. I just ran clang format and modified some other files.
author_association: COLLABORATOR
user: login: jhp-lanl
created_at: 2025-03-05T02:42:47Z
body: Tests are running on our GPU machines
author_association: COLLABORATOR