Repository: numba/numba

Issue:
Overlapping slice copy corrupts array data
Hello.

I'm working in a Miniconda enviroment, using Python+Numpy+Numba code. I implemented a basic nearest-neighbors search in my study work. The fact is that the basic Numpy implementation is working well, but just adding the '@jit' decorator (compiling in Numba), the outputs are differents (it duplicates some neighbors in the end for some unknown reason...)

Here is the basic algorithm:

```
import numpy as np
from numba import jit

@jit(nopython=True)
def knn(p, points, k):
    '''Find the k nearest neighbors (brute force) of the point p
    in the list points (each row is a point)'''

    n = p.size  # Lenght of the points
    M = points.shape[0]  # Number of points
    neighbors = np.zeros((k,n))
    distances = 1e6*np.ones(k)

    for i in xrange(M):
        d = 0
        pt = points[i, :]  # Point to compare
        for r in xrange(n):  # For each coordinate
            aux = p[r] - pt[r]
            d += aux * aux
        if d < distances[k-1]:  # We find a new neighbor
            pos = k-1
            while pos>0 and d<distances[pos-1]:  # Find the position
                pos -= 1
            pt = points[i, :]
            # Insert neighbor and distance:
            neighbors[pos+1:, :] = neighbors[pos:-1, :]
            neighbors[pos, :] = pt
            distances[pos+1:] = distances[pos:-1]
            distances[pos] = d

    return neighbors, distances
```

For testing:

```
p = np.random.rand(10)
points = np.random.rand(250, 10)
k = 5
neighbors, distances = knn(p, points, k)
```

WITHOUT the @jit decorator, one gets the correct answer:

In [1]: distances
Out[1]: array([ 0.3933974 ,  0.44754336,  0.54548715,  0.55619749,  0.5657846 ])

But the Numba compilation gives weird outputs:

Out[2]: distances
Out[2]: array([ 0.3933974 ,  0.44754336,  0.54548715,  **0.54548715,  0.54548715**])

It seems (realized in discussion [here](http://stackoverflow.com/questions/38074352/differences-in-numba-outputs)) that there are problems with the overlapping slice. In Numpy it doesn't causes any problem, but may be Numba translates it to a different procedure.

Thanks you.



Discussion:
this does sound like an issue with overlapping slices.  @pitrou, can you take a look at this?

Right, we don't detect overlapping memory copies. Numpy has some heuristics to make sure this doesn't happen (when it detects the source and destination may overlap, it first makes a private copy of the source).

I think we'll need to support those same heuristics since it looks like people do rely on this behavior.

Yes, our current behaviour really is a bug. It seems Numpy is conservative and just checks the memory bounds (even though there may be "holes" due to slicing).


Resolving patch:
diff --git a/numba/targets/arrayobj.py b/numba/targets/arrayobj.py
--- a/numba/targets/arrayobj.py
+++ b/numba/targets/arrayobj.py
@@ -553,6 +553,13 @@ def get_shape(self):
         """
         raise NotImplementedError
 
+    def get_index_bounds(self):
+        """
+        Return a half-open [lower, upper) range of indices this dimension
+        is guaranteed not to step out of.
+        """
+        raise NotImplementedError
+
     def loop_head(self):
         """
         Start indexation loop.  Return a (index, count) tuple.
@@ -597,6 +604,10 @@ def get_size(self):
     def get_shape(self):
         return (self.size,)
 
+    def get_index_bounds(self):
+        # [0, size)
+        return (self.ll_intp(0), self.size)
+
     def loop_head(self):
         builder = self.builder
         # Initialize loop variable
@@ -637,6 +648,10 @@ def get_size(self):
     def get_shape(self):
         return ()
 
+    def get_index_bounds(self):
+        # [idx, idx+1)
+        return (self.idx, self.builder.add(self.idx, self.get_size()))
+
     def loop_head(self):
         return self.idx, None
 
@@ -671,6 +686,10 @@ def get_size(self):
     def get_shape(self):
         return (self.idx_size,)
 
+    def get_index_bounds(self):
+        # Pessimal heuristic, as we don't want to scan for the min and max
+        return (self.ll_intp(0), self.size)
+
     def loop_head(self):
         builder = self.builder
         # Initialize loop variable
@@ -738,6 +757,11 @@ def get_size(self):
     def get_shape(self):
         return (self.get_size(),)
 
+    def get_index_bounds(self):
+        # Pessimal heuristic, as we don't want to scan for the
+        # first and last true items
+        return (self.ll_intp(0), self.size)
+
     def loop_head(self):
         builder = self.builder
         # Initialize loop variable
@@ -808,6 +832,10 @@ def get_size(self):
     def get_shape(self):
         return (self.get_size(),)
 
+    def get_index_bounds(self):
+        lower, upper = slicing.get_slice_bounds(self.builder, self.slice)
+        return lower, upper
+
     def loop_head(self):
         builder = self.builder
         # Initialize loop variable
@@ -828,7 +856,8 @@ def loop_head(self):
 
     def loop_tail(self):
         builder = self.builder
-        next_index = builder.add(builder.load(self.index), self.slice.step)
+        next_index = builder.add(builder.load(self.index), self.slice.step,
+                                 flags=['nsw'])
         builder.store(next_index, self.index)
         next_count = cgutils.increment_index(builder, builder.load(self.count))
         builder.store(next_count, self.count)
@@ -844,9 +873,10 @@ class FancyIndexer(object):
     def __init__(self, context, builder, aryty, ary, index_types, indices):
         self.context = context
         self.builder = builder
-        self.aryty = ary
+        self.aryty = aryty
         self.shapes = cgutils.unpack_tuple(builder, ary.shape, aryty.ndim)
         self.strides = cgutils.unpack_tuple(builder, ary.strides, aryty.ndim)
+        self.ll_intp = self.context.get_value_type(types.intp)
 
         indexers = []
 
@@ -901,12 +931,48 @@ def __init__(self, context, builder, aryty, ary, index_types, indices):
     def prepare(self):
         for i in self.indexers:
             i.prepare()
+        # Compute the resulting shape
+        self.indexers_shape = sum([i.get_shape() for i in self.indexers], ())
 
     def get_shape(self):
         """
-        Get the resulting shape as Python tuple.
+        Get the resulting data shape as Python tuple.
         """
-        return sum([i.get_shape() for i in self.indexers], ())
+        return self.indexers_shape
+
+    def get_offset_bounds(self, strides, itemsize):
+        """
+        Get a half-open [lower, upper) range of byte offsets spanned by
+        the indexer with the given strides and itemsize.  The indexer is
+        guaranteed to not go past those bounds.
+        """
+        assert len(strides) == self.aryty.ndim
+        builder = self.builder
+        is_empty = cgutils.false_bit
+        zero = self.ll_intp(0)
+        one = self.ll_intp(1)
+        lower = zero
+        upper = zero
+        for indexer, shape, stride in zip(self.indexers, self.indexers_shape,
+                                          strides):
+            is_empty = builder.or_(is_empty,
+                                   builder.icmp_unsigned('==', shape, zero))
+            # Compute [lower, upper) indices on this dimension
+            lower_index, upper_index = indexer.get_index_bounds()
+            lower_offset = builder.mul(stride, lower_index)
+            upper_offset = builder.mul(stride, builder.sub(upper_index, one))
+            # Adjust total interval
+            is_downwards = builder.icmp_signed('<', stride, zero)
+            lower = builder.add(lower,
+                                builder.select(is_downwards, upper_offset, lower_offset))
+            upper = builder.add(upper,
+                                builder.select(is_downwards, lower_offset, upper_offset))
+        # Make interval half-open
+        upper = builder.add(upper, itemsize)
+        # Adjust for empty shape
+        lower = builder.select(is_empty, zero, lower)
+        upper = builder.select(is_empty, zero, upper)
+        return lower, upper
 
     def begin_loops(self):
         indices, counts = zip(*(i.loop_head() for i in self.indexers))
@@ -979,6 +1045,141 @@ def fancy_getitem_array(context, builder, sig, args):
                              aryty, ary, (idxty,), (idx,))
 
 
+def offset_bounds_from_strides(context, builder, arrty, arr, shapes, strides):
+    """
+    Compute a half-open range [lower, upper) of byte offsets from the
+    array's data pointer, that bound the in-memory extent of the array.
+
+    This mimicks offset_bounds_from_strides() from numpy/core/src/private/mem_overlap.c
+    """
+    itemsize = arr.itemsize
+    zero = itemsize.type(0)
+    one = zero.type(1)
+    if arrty.layout in 'CF':
+        # Array is contiguous: contents are laid out sequentially
+        # starting from arr.data and upwards
+        lower = zero
+        upper = builder.mul(itemsize, arr.nitems)
+    else:
+        # Non-contiguous array: need to examine strides
+        lower = zero
+        upper = zero
+        for i in range(arrty.ndim):
+            # Compute the largest byte offset on this dimension
+            #   max_axis_offset = strides[i] * (shapes[i] - 1)
+            # (shapes[i] == 0 is catered for by the empty array case below)
+            max_axis_offset = builder.mul(strides[i],
+                                          builder.sub(shapes[i], one))
+            is_upwards = builder.icmp_signed('>=', max_axis_offset, zero)
+            # Expand either upwards or downwards depending on stride
+            upper = builder.select(is_upwards,
+                                   builder.add(upper, max_axis_offset), upper)
+            lower = builder.select(is_upwards,
+                                   lower, builder.add(lower, max_axis_offset))
+        # Return a half-open range
+        upper = builder.add(upper, itemsize)
+        # Adjust for empty arrays
+        is_empty = builder.icmp_signed('==', arr.nitems, zero)
+        upper = builder.select(is_empty, zero, upper)
+        lower = builder.select(is_empty, zero, lower)
+
+    return lower, upper
+
+
+def compute_memory_extents(context, builder, lower, upper, data):
+    """
+    Given [lower, upper) byte offsets and a base data pointer,
+    compute the memory pointer bounds as pointer-sized integers.
+    """
+    data_ptr_as_int = builder.ptrtoint(data, lower.type)
+    start = builder.add(data_ptr_as_int, lower)
+    end = builder.add(data_ptr_as_int, upper)
+    return start, end
+
+def get_array_memory_extents(context, builder, arrty, arr, shapes, strides, data):
+    """
+    Compute a half-open range [start, end) of pointer-sized integers
+    which fully contain the array data.
+    """
+    lower, upper = offset_bounds_from_strides(context, builder, arrty, arr,
+                                              shapes, strides)
+    return compute_memory_extents(context, builder, lower, upper, data)
+
+
+def extents_may_overlap(context, builder, a_start, a_end, b_start, b_end):
+    """
+    Whether two memory extents [a_start, a_end) and [b_start, b_end)
+    may overlap.
+    """
+    # Comparisons are unsigned, since we are really comparing pointers
+    may_overlap = builder.and_(
+        builder.icmp_unsigned('<', a_start, b_end),
+        builder.icmp_unsigned('<', b_start, a_end),
+        )
+    return may_overlap
+
+
+def maybe_copy_source(context, builder, use_copy,
+                      srcty, src, src_shapes, src_strides, src_data):
+    ptrty = src_data.type
+
+    copy_layout = 'C'
+    copy_data = cgutils.alloca_once_value(builder, src_data)
+    copy_shapes = src_shapes
+    copy_strides = None  # unneeded for contiguous arrays
+
+    with builder.if_then(use_copy, likely=False):
+        # Allocate temporary scratchpad
+        # XXX: should we use a stack-allocated array for very small
+        # data sizes?
+        allocsize = builder.mul(src.itemsize, src.nitems)
+        data = context.nrt_allocate(builder, allocsize)
+        voidptrty = data.type
+        data = builder.bitcast(data, ptrty)
+        builder.store(data, copy_data)
+
+        # Copy source data into scratchpad
+        intp_t = context.get_value_type(types.intp)
+
+        with cgutils.loop_nest(builder, src_shapes, intp_t) as indices:
+            src_ptr = cgutils.get_item_pointer2(builder, src_data,
+                                                src_shapes, src_strides,
+                                                srcty.layout, indices)
+            dest_ptr = cgutils.get_item_pointer2(builder, data,
+                                                 copy_shapes, copy_strides,
+                                                 copy_layout, indices)
+            builder.store(builder.load(src_ptr), dest_ptr)
+
+    def src_getitem(source_indices):
+        assert len(source_indices) == srcty.ndim
+        src_ptr = cgutils.alloca_once(builder, ptrty)
+        with builder.if_else(use_copy, likely=False) as (if_copy, otherwise):
+            with if_copy:
+                builder.store(
+                    cgutils.get_item_pointer2(builder, builder.load(copy_data),
+                                              copy_shapes, copy_strides,
+                                              copy_layout, source_indices,
+                                              wraparound=False),
+                    src_ptr)
+            with otherwise:
+                builder.store(
+                    cgutils.get_item_pointer2(builder, src_data,
+                                              src_shapes, src_strides,
+                                              srcty.layout, source_indices,
+                                              wraparound=False),
+                    src_ptr)
+        return load_item(context, builder, srcty, builder.load(src_ptr))
+
+    def src_cleanup():
+        # Deallocate memory
+        with builder.if_then(use_copy, likely=False):
+            data = builder.load(copy_data)
+            data = builder.bitcast(data, voidptrty)
+            context.nrt_free(builder, data)
+
+    return src_getitem, src_cleanup
+
+
 def fancy_setslice(context, builder, sig, args, index_types, indices):
     """
     Implement slice assignment for arrays.  This implementation works for
@@ -999,11 +1200,11 @@ def fancy_setslice(context, builder, sig, args, index_types, indices):
 
     if isinstance(srcty, types.Buffer):
         # Source is an array
+        src_dtype = srcty.dtype
         src = make_array(srcty)(context, builder, src)
         src_shapes = cgutils.unpack_tuple(builder, src.shape)
         src_strides = cgutils.unpack_tuple(builder, src.strides)
         src_data = src.data
-        src_dtype = srcty.dtype
 
         # Check shapes are equal
         index_shape = indexer.get_shape()
@@ -1018,13 +1219,19 @@ def fancy_setslice(context, builder, sig, args, index_types, indices):
             msg = "cannot assign slice from input of different size"
             context.call_conv.return_user_exc(builder, ValueError, (msg,))
 
-        def src_getitem(source_indices):
-            assert len(source_indices) == srcty.ndim
-            src_ptr = cgutils.get_item_pointer2(builder, src_data,
-                                                src_shapes, src_strides,
-                                                srcty.layout, source_indices,
-                                                wraparound=False)
-            return load_item(context, builder, srcty, src_ptr)
+        # Check for array overlap
+        src_start, src_end = get_array_memory_extents(context, builder, srcty, src,
+                                                      src_shapes, src_strides, src_data)
+
+        dest_lower, dest_upper = indexer.get_offset_bounds(dest_strides, ary.itemsize)
+        dest_start, dest_end = compute_memory_extents(context, builder,
+                                                      dest_lower, dest_upper, dest_data)
+
+        use_copy = extents_may_overlap(context, builder, src_start, src_end, dest_start, dest_end)
+
+        src_getitem, src_cleanup = maybe_copy_source(context, builder, use_copy,
+                                                     srcty, src, src_shapes,
+                                                     src_strides, src_data)
 
     elif isinstance(srcty, types.Sequence):
         src_dtype = srcty.dtype
@@ -1047,6 +1254,9 @@ def src_getitem(source_indices):
                                                 signature(src_dtype, srcty, types.intp))
             return getitem_impl(builder, (src, idx))
 
+        def src_cleanup():
+            pass
+
     else:
         # Source is a scalar (broadcast or not, depending on destination
         # shape).
@@ -1055,6 +1265,9 @@ def src_getitem(source_indices):
         def src_getitem(source_indices):
             return src
 
+        def src_cleanup():
+            pass
+
     # Loop on destination and copy from source to destination
     dest_indices, counts = indexer.begin_loops()
 
@@ -1075,6 +1288,8 @@ def src_getitem(source_indices):
 
     indexer.end_loops()
 
+    src_cleanup()
+
     return context.get_dummy_value()
 
 
@@ -2623,9 +2838,9 @@ def _empty_nd_impl(context, builder, arrtype, shapes):
                 arrtype.layout))
 
     allocsize = builder.mul(itemsize, arrlen)
-    # NOTE: AVX prefer 32-byte alignment
+    align = context.get_preferred_array_alignment(arrtype.dtype)
     meminfo = context.nrt_meminfo_alloc_aligned(builder, size=allocsize,
-                                                align=32)
+                                                align=align)
 
     data = context.nrt_meminfo_data(builder, meminfo)
 
diff --git a/numba/targets/base.py b/numba/targets/base.py
--- a/numba/targets/base.py
+++ b/numba/targets/base.py
@@ -912,6 +912,13 @@ def get_abi_alignment(self, ty):
         assert isinstance(ty, llvmir.Type), "Expected LLVM type"
         return ty.get_abi_alignment(self.target_data)
 
+    def get_preferred_array_alignment(context, ty):
+        """
+        Get preferred array alignment for Numba type *ty*.
+        """
+        # AVX prefers 32-byte alignment
+        return 32
+
     def post_lowering(self, mod, library):
         """Run target specific post-lowering transformation here.
         """
@@ -925,6 +932,30 @@ def _require_nrt(self):
         if not self.enable_nrt:
             raise RuntimeError("Require NRT")
 
+    def nrt_allocate(self, builder, size):
+        """
+        Low-level allocate a new memory area of `size` bytes.
+        """
+        self._require_nrt()
+
+        mod = builder.module
+        fnty = llvmir.FunctionType(void_ptr,
+                                   [self.get_value_type(types.intp)])
+        fn = mod.get_or_insert_function(fnty, name="NRT_Allocate")
+        fn.return_value.add_attribute("noalias")
+        return builder.call(fn, [size])
+
+    def nrt_free(self, builder, ptr):
+        """
+        Low-level free a memory area allocated with nrt_allocate().
+        """
+        self._require_nrt()
+
+        mod = builder.module
+        fnty = llvmir.FunctionType(llvmir.VoidType(), [void_ptr])
+        fn = mod.get_or_insert_function(fnty, name="NRT_Free")
+        return builder.call(fn, [ptr])
+
     def nrt_meminfo_alloc(self, builder, size):
         """
         Allocate a new MemInfo with a data payload of `size` bytes.
diff --git a/numba/targets/slicing.py b/numba/targets/slicing.py
--- a/numba/targets/slicing.py
+++ b/numba/targets/slicing.py
@@ -103,6 +103,24 @@ def get_slice_length(builder, slicestruct):
     return builder.select(is_zero_length, zero, nominal_length)
 
 
+def get_slice_bounds(builder, slicestruct):
+    """
+    Return the [lower, upper) indexing bounds of a slice.
+    """
+    start = slicestruct.start
+    stop = slicestruct.stop
+    zero = start.type(0)
+    one = start.type(1)
+    # This is a bit pessimal, e.g. it will return [1, 5) instead
+    # of [1, 4) for `1:5:2`
+    is_step_negative = builder.icmp_signed('<', slicestruct.step, zero)
+    lower = builder.select(is_step_negative,
+                           builder.add(stop, one), start)
+    upper = builder.select(is_step_negative,
+                           builder.add(start, one), stop)
+    return lower, upper
+
+
 def fix_stride(builder, slice, stride):
     """
     Fix the given stride for the slice's step.


Test patch:
diff --git a/numba/tests/test_overlap.py b/numba/tests/test_overlap.py
new file mode 100644
--- /dev/null
+++ b/numba/tests/test_overlap.py
@@ -0,0 +1,135 @@
+from __future__ import print_function, absolute_import, division
+
+import numpy as np
+
+from numba import unittest_support as unittest
+from numba import jit, types
+from .support import TestCase, tag
+
+
+# Array overlaps involving a displacement
+
+def array_overlap1(src, dest, k=1):
+    assert src.shape == dest.shape
+    dest[k:] = src[:-k]
+
+def array_overlap2(src, dest, k=1):
+    assert src.shape == dest.shape
+    dest[:-k] = src[k:]
+
+def array_overlap3(src, dest, k=1):
+    assert src.shape == dest.shape
+    dest[:,:-k] = src[:,k:]
+
+def array_overlap4(src, dest, k=1):
+    assert src.shape == dest.shape
+    dest[:,k:] = src[:,:-k]
+
+def array_overlap5(src, dest, k=1):
+    assert src.shape == dest.shape
+    dest[...,:-k] = src[...,k:]
+
+def array_overlap6(src, dest, k=1):
+    assert src.shape == dest.shape
+    dest[...,k:] = src[...,:-k]
+
+# Array overlaps involving an in-place reversal
+
+def array_overlap11(src, dest):
+    assert src.shape == dest.shape
+    dest[::-1] = src
+
+def array_overlap12(src, dest):
+    assert src.shape == dest.shape
+    dest[:] = src[::-1]
+
+def array_overlap13(src, dest):
+    assert src.shape == dest.shape
+    dest[:,::-1] = src
+
+def array_overlap14(src, dest):
+    assert src.shape == dest.shape
+    dest[:] = src[:,::-1]
+
+def array_overlap15(src, dest):
+    assert src.shape == dest.shape
+    dest[...,::-1] = src
+
+def array_overlap16(src, dest):
+    assert src.shape == dest.shape
+    dest[:] = src[...,::-1]
+
+
+class TestArrayOverlap(TestCase):
+
+    def check_overlap(self, pyfunc, min_ndim, have_k_argument=False):
+        N = 4
+
+        def vary_layouts(orig):
+            yield orig.copy(order='C')
+            yield orig.copy(order='F')
+            a = orig[::-1].copy()[::-1]
+            assert not a.flags.c_contiguous and not a.flags.f_contiguous
+            yield a
+
+        def check(pyfunc, cfunc, pydest, cdest, kwargs):
+            pyfunc(pydest, pydest, **kwargs)
+            cfunc(cdest, cdest, **kwargs)
+            self.assertPreciseEqual(pydest, cdest)
+
+        cfunc = jit(nopython=True)(pyfunc)
+        # Check for up to 3d arrays
+        for ndim in range(min_ndim, 4):
+            shape = (N,) * ndim
+            orig = np.arange(0, N**ndim).reshape(shape)
+            # Note we cannot copy a 'A' layout array exactly (bitwise),
+            # so instead we call vary_layouts() twice
+            for pydest, cdest in zip(vary_layouts(orig), vary_layouts(orig)):
+                if have_k_argument:
+                    for k in range(1, N):
+                        check(pyfunc, cfunc, pydest, cdest, dict(k=k))
+                else:
+                    check(pyfunc, cfunc, pydest, cdest, {})
+
+    def check_overlap_with_k(self, pyfunc, min_ndim):
+        self.check_overlap(pyfunc, min_ndim=min_ndim, have_k_argument=True)
+
+    def test_overlap1(self):
+        self.check_overlap_with_k(array_overlap1, min_ndim=1)
+
+    def test_overlap2(self):
+        self.check_overlap_with_k(array_overlap2, min_ndim=1)
+
+    def test_overlap3(self):
+        self.check_overlap_with_k(array_overlap3, min_ndim=2)
+
+    def test_overlap4(self):
+        self.check_overlap_with_k(array_overlap4, min_ndim=2)
+
+    def test_overlap5(self):
+        self.check_overlap_with_k(array_overlap5, min_ndim=1)
+
+    def test_overlap6(self):
+        self.check_overlap_with_k(array_overlap6, min_ndim=1)
+
+    def test_overlap11(self):
+        self.check_overlap(array_overlap11, min_ndim=1)
+
+    def test_overlap12(self):
+        self.check_overlap(array_overlap12, min_ndim=1)
+
+    def test_overlap13(self):
+        self.check_overlap(array_overlap13, min_ndim=2)
+
+    def test_overlap14(self):
+        self.check_overlap(array_overlap14, min_ndim=2)
+
+    def test_overlap15(self):
+        self.check_overlap(array_overlap15, min_ndim=1)
+
+    def test_overlap16(self):
+        self.check_overlap(array_overlap16, min_ndim=1)
+
+
+if __name__ == '__main__':
+    unittest.main()