Repository: nv-legate/cunumeric

Issue:
Cunumeric and numpy behave differently for some inplace operations and slicing
In the following function, cunumeric and numpy behave differently. The expected behavior is to obtain an array full of 1, which numpy does:

```python
import cunumeric as num
import numpy as np


def interpolate(data):
    m = (slice(0, -1),)
    p = (slice(1, None),)
    data[p] += data[m] # The issue is caused by this line
    data[p] *= 0.5
    return data[p]


# Cunumeric
data_num = num.ones((20000,))
r_num = interpolate(data_num)
print((r_num == 1.).all()) # False: wrong behavior

# Numpy
data_np = np.ones((20000,))
r_np = interpolate(data_np)
print((r_np == 1.).all()) # True: expected behavior
```

I have observed 3 very different behavior of cunumeric depending on the size of the array:

1.  For small arrays (e.g. shape `(8,)`) cunumeric returns the expected result
2.  For large arrays (e.g. shape `(20000,)`) cunumeric does nor raise any error, but the result array is not full of 1, which is not the expected numpy behavior.
3. For arrays of intermediate sizes (e.g. shape `(10000,)`)) I have a Legion error because of interfering tasks:

```console
[0 - 17246b000]    0.113712 {5}{runtime}: [error 356] LEGION ERROR: Aliased and interfering region requirements for individual tasks are not permitted. Region requirements 0 and 1 of task cunumeric::BinaryOpTask (UID 22) in parent task legion_python_top_level_task (UID 1) are interfering. (from file /Users/charlelie/Documents/RESEARCH/CODES/LEGATE/legate.core/_skbuild/macosx-12.0-arm64-3.10/cmake-build/_deps/legion-src/runtime/legion/legion_tasks.cc:6146)
For more information see:
http://legion.stanford.edu/messages/error_code.html#error_code_356

Signal 6 received by node 0, process 20170 (thread 17246b000) - obtaining backtrace
Signal 6 received by process 20170 (thread 17246b000) at: stack trace: 17 frames
  [0] = 0   libsystem_pthread.dylib             0x00000001a5bfbedf pthread_kill + 287
  [1] = 0   libsystem_c.dylib                   0x00000001a5b3633f abort + 167
  [2] = 0   liblegion.1.dylib                   0x0000000142450807 _ZN6Legion8Internal7Runtime20report_error_messageEiPKciS3_ + 51
  [3] = 0   liblegion.1.dylib                   0x00000001423177f7 _ZThn480_N6Legion8Internal14IndividualTask31report_interfering_requirementsEjj + 223
  [4] = 0   liblegion.1.dylib                   0x0000000142303f3b _ZN6Legion8Internal6TaskOp33perform_intra_task_alias_analysisEbPNS0_11LegionTraceERNSt3__16vectorINS0_14RegionTreePathENS4_9allocatorIS6_EEEE + 2083
  [5] = 0   liblegion.1.dylib                   0x000000014231708f _ZN6Legion8Internal14IndividualTask25trigger_prepipeline_stageEv + 603
  [6] = 0   liblegion.1.dylib                   0x00000001422236e3 _ZN6Legion8Internal9Operation25execute_prepipeline_stageEjb + 407
  [7] = 0   liblegion.1.dylib                   0x0000000142223a47 _ZN6Legion8Internal9Operation27execute_dependence_analysisEv + 51
  [8] = 0   liblegion.1.dylib                   0x000000014215c28f _ZN6Legion8Internal12InnerContext24process_dependence_stageEv + 771
  [9] = 0   liblegion.1.dylib                   0x00000001424f068b _ZN6Legion8Internal7Runtime19legion_runtime_taskEPKvmS3_mN5Realm9ProcessorE + 263
  [10] = 0   librealm.1.dylib                    0x000000014579b07b _ZN5Realm18LocalTaskProcessor12execute_taskEjRKNS_12ByteArrayRefE + 1119
  [11] = 0   librealm.1.dylib                    0x00000001457d802b _ZN5Realm4Task20execute_on_processorENS_9ProcessorE + 471
  [12] = 0   librealm.1.dylib                    0x00000001457deb23 _ZN5Realm25KernelThreadTaskScheduler12execute_taskEPNS_4TaskE + 23
  [13] = 0   librealm.1.dylib                    0x00000001457dd0ff _ZN5Realm21ThreadedTaskScheduler14scheduler_loopEv + 2095
  [14] = 0   librealm.1.dylib                    0x00000001457dd5a7 _ZN5Realm21ThreadedTaskScheduler20scheduler_loop_wlockEv + 39
  [15] = 0   librealm.1.dylib                    0x00000001457e4a0b _ZN5Realm12KernelThread13pthread_entryEPv + 499
  [16] = 0   libsystem_pthread.dylib             0x00000001a5bfc26b _pthread_start + 147
```

I think this was also reported in #57 .


Discussion:
On the other hand, I am using other in-place operators with overlap between the input and output region, without noticing any issue. For example:

```python
num.einsum("ij...,ij...->i...", x, y, out=x[:, 0, ...])
```

So, I am wondering if such in-place operation only works for some operators and the bug only occurs for some others.

Resolving patch:
diff --git a/cunumeric/deferred.py b/cunumeric/deferred.py
--- a/cunumeric/deferred.py
+++ b/cunumeric/deferred.py
@@ -296,6 +296,7 @@ def _copy_if_overlapping(self, other: DeferredArray) -> DeferredArray:
             self.runtime.create_empty_thunk(
                 self.shape,
                 self.base.type,
+                inputs=[self],
             ),
         )
         copy.copy(self, deep=True)
@@ -1098,22 +1099,20 @@ def set_item(self, key: Any, rhs: Any) -> None:
                 # to set the result back. In cuNumeric, the object we
                 # return in step (1) is actually a subview to the array arr
                 # through which we make updates in place, so after step (2) is
-                # done, # the effect of inplace update is already reflected
+                # done, the effect of inplace update is already reflected
                 # to the arr. Therefore, we skip the copy to avoid redundant
                 # copies if we know that we hit such a scenario.
                 # TODO: We should make this work for the advanced indexing case
-                if view.base == rhs.base:
+                # NOTE: Neither Store nor Storage have an __eq__, so we can
+                # only check that the underlying RegionField/Future corresponds
+                # to the same Legion handle.
+                if (
+                    view.base.has_storage
+                    and rhs.base.has_storage
+                    and view.base.storage.same_handle(rhs.base.storage)
+                ):
                     return
 
-                if view.base.overlaps(rhs.base):
-                    rhs_copy = self.runtime.create_empty_thunk(
-                        rhs.shape,
-                        rhs.base.type,
-                        inputs=[rhs],
-                    )
-                    rhs_copy.copy(rhs, deep=False)
-                    rhs = rhs_copy
-
                 view.copy(rhs, deep=False)
 
     def broadcast_to(self, shape: NdShape) -> NumPyThunk:
@@ -1870,6 +1869,9 @@ def put(self, indices: Any, values: Any, check_bounds: bool) -> None:
 
         assert indices.size == values.size
 
+        # Handle store overlap
+        values = values._copy_if_overlapping(self_tmp)
+
         # first, we create indirect array with PointN type that
         # (indices.size,) shape and is used to copy data from values
         # to the target ND array (self)
@@ -1910,11 +1912,12 @@ def put(self, indices: Any, values: Any, check_bounds: bool) -> None:
     @auto_convert("mask", "values")
     def putmask(self, mask: Any, values: Any) -> None:
         assert self.shape == mask.shape
-
+        values = values._copy_if_overlapping(self)
         if values.shape != self.shape:
             values_new = values._broadcast(self.shape)
         else:
             values_new = values.base
+
         task = self.context.create_auto_task(CuNumericOpCode.PUTMASK)
         task.add_input(self.base)
         task.add_input(mask.base)
@@ -3142,6 +3145,7 @@ def unary_op(
         multiout: Optional[Any] = None,
     ) -> None:
         lhs = self.base
+        src = src._copy_if_overlapping(self)
         rhs = src._broadcast(lhs.shape)
 
         with Annotation({"OpCode": op.name}):
@@ -3304,7 +3308,9 @@ def binary_op(
         args: Any,
     ) -> None:
         lhs = self.base
+        src1 = src1._copy_if_overlapping(self)
         rhs1 = src1._broadcast(lhs.shape)
+        src2 = src2._copy_if_overlapping(self)
         rhs2 = src2._broadcast(lhs.shape)
 
         with Annotation({"OpCode": op_code.name}):


Test patch:
diff --git a/tests/integration/test_overlap.py b/tests/integration/test_overlap.py
new file mode 100644
--- /dev/null
+++ b/tests/integration/test_overlap.py
@@ -0,0 +1,80 @@
+# Copyright 2024 NVIDIA Corporation
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+import numpy as np
+import pytest
+from utils.generators import mk_seq_array
+
+import cunumeric as num
+
+
+def setitem(lib, a, slice_lhs, slice_rhs):
+    a[slice_lhs] = a[slice_rhs]
+
+
+def dot(lib, a, slice_lhs, slice_rhs):
+    modes = "".join([chr(ord("a") + m) for m in range(len(a.shape))])
+    expr = f"{modes},{modes}->{modes}"
+    lib.einsum(expr, a[slice_lhs], a[slice_rhs], out=a[slice_lhs])
+
+
+def unary_arith(lib, a, slice_lhs, slice_rhs):
+    lib.sin(a[slice_rhs], out=a[slice_lhs])
+
+
+def binary_arith(lib, a, slice_lhs, slice_rhs):
+    a[slice_lhs] += a[slice_rhs]
+
+
+def put(lib, a, slice_lhs, slice_rhs):
+    indices = lib.flip(lib.arange(a[slice_rhs].size))
+    a[slice_lhs].put(indices, a[slice_rhs])
+
+
+def putmask(lib, a, slice_lhs, slice_rhs):
+    mask = (mk_seq_array(lib, a[slice_rhs].shape) % 2).astype(bool)
+    lib.putmask(a[slice_lhs], mask, a[slice_rhs])
+
+
+SHAPES = ((4,), (4, 5), (4, 5, 6))
+OPERATIONS = (setitem, dot, unary_arith, binary_arith, put, putmask)
+
+
+@pytest.mark.parametrize("partial", (True, False))
+@pytest.mark.parametrize("shape", SHAPES)
+@pytest.mark.parametrize("operation", OPERATIONS)
+def test_partial(partial, shape, operation):
+    if partial:
+        # e.g. for shape = (4,5) and setitem: lhs[1:,:] = rhs[:-1,:]
+        slice_lhs = (slice(1, None),) + (slice(None),) * (len(shape) - 1)
+        slice_rhs = (slice(None, -1),) + (slice(None),) * (len(shape) - 1)
+    else:
+        # e.g. for shape = (4,5) and setitem: lhs[:,:] = rhs[:,:]
+        slice_lhs = (slice(None),) * len(shape)
+        slice_rhs = (slice(None),) * len(shape)
+
+    a_np = mk_seq_array(np, shape).astype(np.float64)
+    a_num = mk_seq_array(num, shape).astype(np.float64)
+
+    operation(np, a_np, slice_lhs, slice_rhs)
+    operation(num, a_num, slice_lhs, slice_rhs)
+
+    assert np.array_equal(a_np, a_num)
+
+
+if __name__ == "__main__":
+    import sys
+
+    sys.exit(pytest.main(sys.argv))