Repository: numba/numba

Issue:
Bug in box/unboxing of lists in JIT mode
This is the track the following. It appears that the use of lists in JITed functions is reliant on a boxing/unboxing mechanism that is not numerically consistent or safe.

Example:

```
import numpy as np
import numba as nb

d1=list([np.float64(1),np.float32(2),np.int64(3),np.complex128(4+5j)])
d2=list([np.complex128(4+5j),np.float64(1),np.float32(2),np.int64(3)])

def foo(items):
    acc = 0
    for k in items:
        acc = acc + k
    return (acc, items)

@nb.jit
def foo_jit(items):
    acc = 0
    for k in items:
        acc = acc + k
    return (acc, items)

def run(case, f, fjit, v):
    print("__Case ",case,"__\nInput:",v)
    print("\nWithout JIT:\nResult:", f(v), "\nInput vector: ", v)
    print("\nWith JIT:\nResult:", fjit(v), "\nInput vector: ", v, "\n")

run(1, foo, foo_jit, d1)
run(2, foo, foo_jit, d2)
```

Gives output:

```
$ python bug.py 
__Case  1 __
Input: [1.0, 2.0, 3, (4+5j)]

Without JIT:
Result: ((10+5j), [1.0, 2.0, 3, (4+5j)]) 
Input vector:  [1.0, 2.0, 3, (4+5j)]
bug2.py:23: ComplexWarning: Casting complex values to real discards the imaginary part
  print("\nWith JIT:\nResult:", fjit(v), "\nInput vector: ", v, "\n")

With JIT:
Result: (10.0, [1.0, 2.0, 3.0, 4.0]) 
Input vector:  [1.0, 2.0, 3.0, 4.0] 

__Case  2 __
Input: [(4+5j), 1.0, 2.0, 3]

Without JIT:
Result: ((10+5j), [(4+5j), 1.0, 2.0, 3]) 
Input vector:  [(4+5j), 1.0, 2.0, 3]

With JIT:
Result: ((10+5j), [(4+5j), (1+0j), (2+0j), (3+0j)]) 
Input vector:  [(4+5j), (1+0j), (2+0j), (3+0j)] 
```

It would appear that the lists, when used in the JITed functions, are considered homogeneous in the type of the first element in the list. This leads to unfortunate casting in the first case where a complex value has its imaginary part truncated (and the runtime warns). In the second case the first element is a `complex128` type and therefore all casts are safe in both domain and precision, although not necessarily desirable for performance. In the first case, due to the incorrect interpretation of the list the function returns an incorrect result. Further the JITed code is mutating an argument (the list) and it probably shouldn't be doing so as there are no stores made to the list in the function.



Discussion:
Issue #1544 deals with the first part, i.e. casting of elements and limitation to homogeneous lists. Also, the docs, namely "Build-in types / list"  in `pysupported.rst`, currently do not state the limitation to homogeneous types and need to be updated.

The second part, i.e. the parameter being mutated, is even worse. Here is a smaller example:

``` python
from numba import njit

def is_homogeneous(items):
  return all(isinstance(x, type(items[0])) for x in items)

def do_nothing(items):
  return

d = [1, 2.0, (1+0j)]
assert not is_homogeneous(d)
do_nothing(d)
assert not is_homogeneous(d)
njit()(do_nothing)(d)
assert not is_homogeneous(d)  # throws "TypeError: can't convert complex to int"
assert not is_homogeneous(d)  # throws "AssertionError"
```


Resolving patch:
diff --git a/numba/pythonapi.py b/numba/pythonapi.py
--- a/numba/pythonapi.py
+++ b/numba/pythonapi.py
@@ -202,6 +202,11 @@ def decref(self, obj):
         fn = self._get_function(fnty, name="Py_DecRef")
         self.builder.call(fn, [obj])
 
+    def get_type(self, obj):
+        fnty = Type.function(self.pyobj, [self.pyobj])
+        fn = self._get_function(fnty, name="numba_py_type")
+        return self.builder.call(fn, [obj])
+
     #
     # Argument unpacking
     #
diff --git a/numba/targets/boxing.py b/numba/targets/boxing.py
--- a/numba/targets/boxing.py
+++ b/numba/targets/boxing.py
@@ -454,6 +454,57 @@ def box_list(typ, val, c):
     return c.builder.load(res)
 
 
+def _python_list_to_native(typ, obj, c, size, listptr, errorptr):
+    """
+    Construct a new native list from a Python list.
+    """
+    # Allocate a new native list
+    ok, list = listobj.ListInstance.allocate_ex(c.context, c.builder, typ, size)
+    with c.builder.if_else(ok, likely=True) as (if_ok, if_not_ok):
+        with if_ok:
+            list.size = size
+            zero = ir.Constant(size.type, 0)
+            with c.builder.if_then(c.builder.icmp_signed('>', size, zero),
+                                   likely=True):
+                # Traverse Python list and unbox objects into native list
+                expected_typobj = c.pyapi.get_type(c.pyapi.list_getitem(obj, zero))
+                with cgutils.for_range(c.builder, size) as loop:
+                    itemobj = c.pyapi.list_getitem(obj, loop.index)
+                    typobj = c.pyapi.get_type(itemobj)
+
+                    # Mandate that objects all have the same exact type
+                    type_mismatch = c.builder.icmp_signed('!=', typobj,
+                                                          expected_typobj)
+                    with c.builder.if_then(type_mismatch, likely=False):
+                        c.builder.store(cgutils.true_bit, errorptr)
+                        c.pyapi.err_set_string("PyExc_TypeError",
+                                               "can't unbox heterogenous list")
+                        loop.do_break()
+
+                    # XXX we don't call native cleanup for each
+                    # list element, since that would require keeping
+                    # of which unboxings have been successful.
+                    native = c.unbox(typ.dtype, itemobj)
+                    with c.builder.if_then(native.is_error, likely=False):
+                        c.builder.store(cgutils.true_bit, errorptr)
+                    list.setitem(loop.index, native.value)
+
+            if typ.reflected:
+                list.parent = obj
+            # Stuff meminfo pointer into the Python object for
+            # later reuse.
+            c.pyapi.list_set_private_data(obj, list.meminfo)
+            list.set_dirty(False)
+            c.builder.store(list.value, listptr)
+
+        with if_not_ok:
+            c.builder.store(cgutils.true_bit, errorptr)
+
+    # If an error occurred, drop the whole native list
+    with c.builder.if_then(c.builder.load(errorptr)):
+        c.context.nrt_decref(c.builder, typ, list.value)
+
+
 @unbox(types.List)
 def unbox_list(typ, obj, c):
     """
@@ -483,32 +534,7 @@ def unbox_list(typ, obj, c):
             c.builder.store(list.value, listptr)
 
         with otherwise:
-            # Allocate a new native list
-            ok, list = listobj.ListInstance.allocate_ex(c.context, c.builder, typ, size)
-            with c.builder.if_else(ok, likely=True) as (if_ok, if_not_ok):
-                with if_ok:
-                    list.size = size
-                    with cgutils.for_range(c.builder, size) as loop:
-                        itemobj = c.pyapi.list_getitem(obj, loop.index)
-                        # XXX we don't call native cleanup for each
-                        # list element, since that would require keeping
-                        # of which unboxings have been successful.
-                        native = c.unbox(typ.dtype, itemobj)
-                        with c.builder.if_then(native.is_error, likely=False):
-                            c.builder.store(cgutils.true_bit, errorptr)
-                        list.setitem(loop.index, native.value)
-                    if typ.reflected:
-                        list.parent = obj
-                    # Stuff meminfo pointer into the Python object for
-                    # later reuse.
-                    c.pyapi.list_set_private_data(obj, list.meminfo)
-                    c.builder.store(list.value, listptr)
-                with if_not_ok:
-                    c.builder.store(cgutils.true_bit, errorptr)
-
-            # If an error occurred, drop the whole native list
-            with c.builder.if_then(c.builder.load(errorptr)):
-                c.context.nrt_decref(c.builder, typ, list.value)
+            _python_list_to_native(typ, obj, c, size, listptr, errorptr)
 
     def cleanup():
         # Clean up the stuffed pointer, as the meminfo is now invalid.


Test patch:
diff --git a/numba/tests/test_lists.py b/numba/tests/test_lists.py
--- a/numba/tests/test_lists.py
+++ b/numba/tests/test_lists.py
@@ -742,7 +742,7 @@ def check(arg):
     def test_numbers(self):
         check = self.check_unary(unbox_usecase)
         check([1, 2])
-        check([1j, 2.5])
+        check([1j, 2.5j])
 
     def test_tuples(self):
         check = self.check_unary(unbox_usecase2)
@@ -759,10 +759,16 @@ def test_list_of_tuples_inside_tuple(self):
         check((1, [(2,), (3,)]))
 
     def test_errors(self):
-        # See #1545: error checking should ensure the list is homogenous
-        msg = "can't convert complex to (int|long)"
+        # See #1545 and #1594: error checking should ensure the list is
+        # homogenous
+        msg = "can't unbox heterogenous list"
         pyfunc = noop
         cfunc = jit(nopython=True)(pyfunc)
+        lst = [1, 2.5]
+        with self.assert_type_error(msg):
+            cfunc(lst)
+        # The list hasn't been changed (bogus reflecting)
+        self.assertEqual(lst, [1, 2.5])
         with self.assert_type_error(msg):
             cfunc([1, 2j])
         # Same when the list is nested in a tuple or namedtuple
@@ -827,6 +833,17 @@ def test_reflect_same_list(self):
         self.assertPreciseEqual(pylist, clist)
         self.assertPreciseEqual(sys.getrefcount(pylist), sys.getrefcount(clist))
 
+    def test_reflect_clean(self):
+        """
+        When the list wasn't mutated, no reflection should take place.
+        """
+        cfunc = jit(nopython=True)(noop)
+        # Use a complex, as Python integers can be cached
+        l = [12.5j]
+        ids = [id(x) for x in l]
+        cfunc(l)
+        self.assertEqual([id(x) for x in l], ids)
+
 
 if __name__ == '__main__':
     unittest.main()