Repository: numba/numba

Issue:
invalid result on master when aggregating flat array with size=0
``` python
import numpy as np
from numba import guvectorize

@guvectorize(['void(float64[:, :], float64[:])'],
              '(x, y)->()')
def my_sum(a, out):
    asum = 0.0
    for ai in a.flat:
        asum += ai
    out[0] = asum

x = np.arange(0.0).reshape(2, 0)
print repr(x)
print repr(my_sum(x)) 
```

```
array([], shape=(2, 0), dtype=float64)
1.7272338168659652e-77
```

I'm running the master version of numba (actually 0.15.1-28-g2f5d025) and numpy 1.9.0 on Python 2.7/OSX (via conda).

This is indeed pathological data, but this part of the bottleneck test suite failed when I tested it for [numbagg](https://github.com/shoyer/numbagg).



Discussion:
Is it specific to the use of `guvectorize`? Or does it also apply to normal jitted functions?

@pitrou Good question. Here is my `jit` version, which for which I cannot reproduce the issue:

```
import numpy as np
from numba import jit

@jit
def my_sum(a):
    asum = 0.0
    for ai in a.flat:
        asum += ai
    return asum

x = np.arange(0.0).reshape(2, 0)
print repr(x)
print repr(my_sum(x)) 
```

This maybe a numpy bug.  Consider:

``` python
import numpy as np
import numpy.core.umath_tests as ut

x = np.arange(0.0).reshape(2, 0)
y = np.arange(0.0).reshape(0, 2)
print(x, x.shape, x.data.nbytes)
print(y, y.shape, y.data.nbytes)
print(ut.matrix_multiply(x, y))
```

which prints:

```
[] (2, 0) 0
[] (0, 2) 0
[[  1.72723371e-077   1.72723371e-077]
 [  2.12435947e-314   2.78134365e-309]]
```

Interesting. I should note that this _does_ work on the numba 0.15.1 (I accidentally omitted that in my original report), so it is likely related to work refactoring `.flat`.

Also might be related to this numpy issue: https://github.com/numpy/numpy/issues/5195

The fact that it works with `@jit` is a bit fishy. It would be nice to know what the `a` array looks like inside the `@guvectorize` version.

@pitrou I tried inserting some print statements, but of course that causes the function to be executed in python mode, which removes the bug.

If you use the print() function and only print integers and floats, the function should be able to compile in nopython mode (add nopython=True to the jit() call to ensure this).

OK, perhaps this output may be helpful:

``` python
import numpy as np
from numba import guvectorize

@guvectorize(['void(float64[:, :], float64[:])'],
              '(x, y)->()', nopython=True)
def my_sum(a, out):
    asum = 0.0
    print(asum)
    adding = 99999999 # some sentinel value
    for n, ai in enumerate(a.flat):
        print(adding)
        print(ai)
        asum += ai
    print(asum)
    out[0] = asum

for shape in [(0, 0), (0, 2), (2, 0)]:
    print('\nshape {}:'.format(shape))
    x = np.arange(0.0).reshape(shape)
    print repr(my_sum(x)) 
```

```
shape (0, 0):
0.0
0.0
0.0

shape (0, 2):
0.0
99999999
0.0
99999999
3.10503636984e+231
3.10503636984e+231
3.1050363698376418e+231

shape (2, 0):
0.0
99999999
1.32768739285e-315
1.32768739285e-315
1.3276873928472899e-315
```

It looks like `.flat` is sometimes reading from unallocated memory.

Thank you. Yes, it's an issue with `.flat`. It only manifests on the code path for non-contigouous arrays (you can also trigger it with explicit typing in the jit() call, e.g. `@jit('void(float64[:, :])', nopython=True)`. The problem is that if one dimension is zero-sized (but not all), the iteration logic becomes incorrect. IMO the best fix will be to add a special path if there is a zero-sized dimension, since the iterator is empty then.


Resolving patch:
diff --git a/numba/targets/arrayobj.py b/numba/targets/arrayobj.py
--- a/numba/targets/arrayobj.py
+++ b/numba/targets/arrayobj.py
@@ -601,6 +601,7 @@ class FlatIter(cgutils.Structure):
             _fields = [('array', types.CPointer(array_type)),
                        ('pointers', types.CPointer(types.CPointer(dtype))),
                        ('indices', types.CPointer(types.intp)),
+                       ('empty', types.CPointer(types.boolean)),
                        ]
 
             def init_specific(self, context, builder, arrty, arr):
@@ -608,6 +609,8 @@ def init_specific(self, context, builder, arrty, arr):
                 one = context.get_constant(types.intp, 1)
                 data = arr.data
                 ndim = arrty.ndim
+                shapes = cgutils.unpack_tuple(builder, arr.shape, ndim)
+
                 indices = cgutils.alloca_once(builder, zero.type,
                                               size=context.get_constant(types.intp,
                                                                         arrty.ndim))
@@ -615,6 +618,7 @@ def init_specific(self, context, builder, arrty, arr):
                                                size=context.get_constant(types.intp,
                                                                          arrty.ndim))
                 strides = cgutils.unpack_tuple(builder, arr.strides, ndim)
+                empty = cgutils.alloca_once_value(builder, cgutils.false_byte)
 
                 # Initialize each dimension with the next index and pointer
                 # values.  For the last (inner) dimension, this is 0 and the
@@ -630,9 +634,17 @@ def init_specific(self, context, builder, arrty, arr):
                         p = cgutils.pointer_add(builder, data, strides[dim])
                         builder.store(p, ptrptr)
                         builder.store(one, idxptr)
+                    # 0-sized dimensions really indicate an empty array,
+                    # but we have to catch that condition early to avoid
+                    # a bug inside the iteration logic (see issue #846).
+                    dim_size = shapes[dim]
+                    dim_is_empty = builder.icmp(lc.ICMP_EQ, dim_size, zero)
+                    with cgutils.if_unlikely(builder, dim_is_empty):
+                        builder.store(cgutils.true_byte, empty)
 
                 self.indices = indices
                 self.pointers = pointers
+                self.empty = empty
 
             def iternext_specific(self, context, builder, arrty, arr, result):
                 ndim = arrty.ndim
@@ -650,6 +662,12 @@ def iternext_specific(self, context, builder, arrty, arr, result):
                 bbcont = cgutils.append_basic_block(builder, 'continued')
                 bbend = cgutils.append_basic_block(builder, 'end')
 
+                # Catch already computed iterator exhaustion
+                is_empty = cgutils.as_bool_bit(builder, builder.load(self.empty))
+                with cgutils.if_unlikely(builder, is_empty):
+                    result.set_valid(False)
+                    builder.branch(bbend)
+
                 # Current pointer inside last dimension
                 last_ptr = cgutils.alloca_once(builder, data.type)
 


Test patch:
diff --git a/numba/tests/test_array_methods.py b/numba/tests/test_array_methods.py
--- a/numba/tests/test_array_methods.py
+++ b/numba/tests/test_array_methods.py
@@ -1,8 +1,11 @@
 from __future__ import division
+
+import numpy as np
+
 from numba import unittest_support as unittest
-from numba import typeof
+from numba import typeof, types
 from numba.compiler import compile_isolated
-import numpy as np
+from .support import TestCase
 
 
 def array_sum(arr):
@@ -25,8 +28,15 @@ def array_flat(arr, out):
     for i, v in enumerate(arr.flat):
         out[i] = v
 
+def array_flat_sum(arr):
+    s = 0
+    for i, v in enumerate(arr.flat):
+        s = s + (i + 1) * v
+    return s
+
+
+class TestArrayMethods(TestCase):
 
-class TestArrayMethods(unittest.TestCase):
     def test_array_sum_int_1d(self):
         arr = np.arange(10, dtype=np.int32)
         arrty = typeof(arr)
@@ -95,11 +105,13 @@ def test_array_sum_float_3d_any(self):
 
         np.testing.assert_allclose(arr.sum(), cfunc(arr))
 
-    def check_array_flat(self, arr):
+    def check_array_flat(self, arr, arrty=None):
         out = np.zeros(arr.size, dtype=arr.dtype)
         nb_out = out.copy()
+        if arrty is None:
+            arrty = typeof(arr)
 
-        cres = compile_isolated(array_flat, [typeof(arr), typeof(out)])
+        cres = compile_isolated(array_flat, [arrty, typeof(out)])
         cfunc = cres.entry_point
 
         array_flat(arr, out)
@@ -107,6 +119,12 @@ def check_array_flat(self, arr):
 
         self.assertTrue(np.all(out == nb_out), (out, nb_out))
 
+    def check_array_flat_sum(self, arr, arrty):
+        cres = compile_isolated(array_flat_sum, [arrty])
+        cfunc = cres.entry_point
+
+        self.assertPreciseEqual(cfunc(arr), array_flat_sum(arr))
+
     def test_array_flat_3d(self):
         arr = np.arange(24).reshape(4, 2, 3)
 
@@ -129,6 +147,25 @@ def test_array_flat_3d(self):
         self.assertEqual(typeof(arr).layout, 'A')
         self.check_array_flat(arr)
 
+    def test_array_flat_empty(self):
+        # Test .flat() with various shapes of empty arrays, contiguous
+        # and non-contiguous (see issue #846).
+        arr = np.zeros(0, dtype=np.int32)
+        arr = arr.reshape(0, 2)
+        arrty = types.Array(types.int32, 2, layout='C')
+        self.check_array_flat_sum(arr, arrty)
+        arrty = types.Array(types.int32, 2, layout='F')
+        self.check_array_flat_sum(arr, arrty)
+        arrty = types.Array(types.int32, 2, layout='A')
+        self.check_array_flat_sum(arr, arrty)
+        arr = arr.reshape(2, 0)
+        arrty = types.Array(types.int32, 2, layout='C')
+        self.check_array_flat_sum(arr, arrty)
+        arrty = types.Array(types.int32, 2, layout='F')
+        self.check_array_flat_sum(arr, arrty)
+        arrty = types.Array(types.int32, 2, layout='A')
+        self.check_array_flat_sum(arr, arrty)
+
     def test_array_sum_global(self):
         arr = np.arange(10, dtype=np.int32)
         arrty = typeof(arr)