Repository: numba/numba

Issue:
Heap corruption of Lists of numpy arrays
Lists of numpy arrays (at least of float 1D type) cannot be brought to length zero without heap corruption. Best guess at this point: refcount issue and double free of allocated memory.

Also, Lists of numpy scalars for some reason do not suffer this issue.

Minimal reproducer:

```python
import numpy as np
from numba import  float64
from numba.typed import List

float_array = float64[:]

l = List.empty_list(float_array)
l.append(np.ones(3,dtype=np.float64))
l.pop()
````


Discussion:
@Viissataa thanks for reporting this on the Numba issue tracker. I have reproduced this locally and can confirm, that it is likely a bug.
After a bit of debugging, this is likely a refcounting issue, essentially a double free is the root cause of the SIGSEGV.

Resolving patch:
diff --git a/numba/typed/listobject.py b/numba/typed/listobject.py
--- a/numba/typed/listobject.py
+++ b/numba/typed/listobject.py
@@ -661,24 +661,11 @@ def handle_slice(l, s):
 
 @intrinsic
 def _list_getitem(typingctx, l, index):
-    return _list_getitem_pop_helper(typingctx, l, index, 'getitem')
-
-
-@intrinsic
-def _list_pop(typingctx, l, index):
-    return _list_getitem_pop_helper(typingctx, l, index, 'pop')
-
-
-def _list_getitem_pop_helper(typingctx, l, index, op):
-    """Wrap numba_list_getitem and numba_list_pop
+    """Wrap numba_list_getitem
 
     Returns 2-tuple of (int32, ?item_type)
 
-    This is a helper that is parametrized on the type of operation, which can
-    be either 'pop' or 'getitem'. This is because, signature wise, getitem and
-    pop and are the same.
     """
-    assert(op in ("pop", "getitem"))
     IS_NOT_NONE = not isinstance(l.item_type, types.NoneType)
     resty = types.Tuple([types.int32,
                          types.Optional(l.item_type
@@ -694,7 +681,7 @@ def codegen(context, builder, sig, args):
         [tl, tindex] = sig.args
         [l, index] = args
         fn = builder.module.get_or_insert_function(
-            fnty, name='numba_list_{}'.format(op))
+            fnty, name='numba_list_getitem')
 
         dm_item = context.data_model_manager[tl.item_type]
         ll_item = context.get_data_type(tl.item_type)
@@ -900,21 +887,38 @@ def impl_pop(l, index=-1):
         def impl(l, index=-1):
             if len(l) == 0:
                 raise IndexError("pop from empty list")
-            index = handle_index(l, index)
-            castedindex = _cast(index, indexty)
-            status, item = _list_pop(l, castedindex)
-            if status == ListStatus.LIST_OK:
-                return _nonoptional(item)
-            elif status == ListStatus.LIST_ERR_IMMUTABLE:
-                raise ValueError("list is immutable")
-            else:
-                raise AssertionError("internal list error during pop")
+            cindex = _cast(handle_index(l, index), indexty)
+            item = l[cindex]
+            del l[cindex]
+            return item
         return impl
 
     else:
         raise TypingError("argument for pop must be an integer")
 
 
+@intrinsic
+def _list_delitem(typingctx, l, index):
+    resty = types.int32
+    sig = resty(l, index)
+
+    def codegen(context, builder, sig, args):
+        fnty = ir.FunctionType(
+            ll_status,
+            [ll_list_type, ll_ssize_t],
+        )
+        [tl, tindex] = sig.args
+        [l, index] = args
+        fn = builder.module.get_or_insert_function(
+            fnty, name='numba_list_delitem')
+
+        lp = _container_get_data(context, builder, tl, l)
+        status = builder.call(fn, [lp, index])
+        return status
+
+    return sig, codegen
+
+
 @intrinsic
 def _list_delete_slice(typingctx, l, start, stop, step):
     """Wrap numba_list_delete_slice
@@ -952,10 +956,18 @@ def impl_delitem(l, index):
     if not isinstance(l, types.ListType):
         return
 
+    _check_for_none_typed(l, 'delitem')
+
     if index in index_types:
         def integer_impl(l, index):
-            l.pop(index)
-
+            cindex = _cast(handle_index(l, index), INDEXTY)
+            status = _list_delitem(l, cindex)
+            if status == ListStatus.LIST_OK:
+                return
+            elif status == ListStatus.LIST_ERR_IMMUTABLE:
+                raise ValueError("list is immutable")
+            else:
+                raise AssertionError("internal list error during delitem")
         return integer_impl
 
     elif isinstance(index, types.SliceType):
@@ -1123,7 +1135,7 @@ def impl(l, item):
         casteditem = _cast(item, itemty)
         for i, n in enumerate(l):
             if casteditem == n:
-                l.pop(i)
+                del l[i]
                 return
         else:
             raise ValueError("list.remove(x): x not in list")
@@ -1138,7 +1150,7 @@ def impl_clear(l):
 
     def impl(l):
         while len(l):
-            l.pop()
+            del l[-1]
 
     return impl
 


Test patch:
diff --git a/numba/tests/test_listimpl.py b/numba/tests/test_listimpl.py
--- a/numba/tests/test_listimpl.py
+++ b/numba/tests/test_listimpl.py
@@ -55,6 +55,15 @@ def __iter__(self):
     def __delitem__(self, i):
         self.list_delitem(i)
 
+    def handle_index(self, i):
+        # handling negative indices is done at the compiler level, so we only
+        # support -1 to be last element of the list here
+        if i < -1 or len(self) == 0:
+            IndexError("list index out of range")
+        elif i == -1:
+            i = len(self) - 1
+        return i
+
     @property
     def allocated(self):
         return self.list_allocated()
@@ -107,6 +116,7 @@ def list_setitem(self, i, item):
             self.tc.assertEqual(status, LIST_OK)
 
     def list_getitem(self, i):
+        i = self.handle_index(i)
         item_out_buffer = ctypes.create_string_buffer(self.item_size)
         status = self.tc.numba_list_getitem(self.lp, i, item_out_buffer)
         if status == LIST_ERR_INDEX:
@@ -122,21 +132,11 @@ def list_append(self, item):
         self.tc.assertEqual(status, LIST_OK)
 
     def list_pop(self, i):
-        # handling negative indices is done at the compiler level, so we only
-        # support -1 to be last element of the list here
-        if i < -1 or len(self) == 0:
-            IndexError("list index out of range")
-        elif i == -1:
-            i = len(self) - 1
-        item_out_buffer = ctypes.create_string_buffer(self.item_size)
-        status = self.tc.numba_list_pop(self.lp, i, item_out_buffer)
-        if status == LIST_ERR_INDEX:
-            raise IndexError("list index out of range")
-        elif status == LIST_ERR_IMMUTABLE:
-            raise ValueError("list is immutable")
-        else:
-            self.tc.assertEqual(status, LIST_OK)
-            return item_out_buffer.raw
+        # pop is getitem and delitem
+        i = self.handle_index(i)
+        item = self.list_getitem(i)
+        self.list_delitem(i)
+        return item
 
     def list_delitem(self, i):
         # special case slice
@@ -148,9 +148,15 @@ def list_delitem(self, i):
             if status == LIST_ERR_IMMUTABLE:
                 raise ValueError("list is immutable")
             self.tc.assertEqual(status, LIST_OK)
-        # must be an integer, defer to pop
+        # must be an integer, defer to delitem
         else:
-            self.list_pop(i)
+            i = self.handle_index(i)
+            status = self.tc.numba_list_delitem(self.lp, i)
+            if status == LIST_ERR_INDEX:
+                raise IndexError("list index out of range")
+            elif status == LIST_ERR_IMMUTABLE:
+                raise ValueError("list is immutable")
+            self.tc.assertEqual(status, LIST_OK)
 
     def list_iter(self, itptr):
         self.tc.numba_list_iter(itptr, self.lp)
@@ -260,11 +266,11 @@ def wrap(name, restype, argtypes=()):
             ctypes.c_int,
             [list_t, ctypes.c_ssize_t, ctypes.c_char_p],
         )
-        # numba_list_pop(NB_List *l,  Py_ssize_t i, char *out)
-        self.numba_list_pop = wrap(
-            'list_pop',
+        # numba_list_delitem(NB_List *l,  Py_ssize_t i)
+        self.numba_list_delitem = wrap(
+            'list_delitem',
             ctypes.c_int,
-            [list_t, ctypes.c_ssize_t, ctypes.c_char_p],
+            [list_t, ctypes.c_ssize_t],
         )
         # numba_list_delete_slice(NB_List *l,
         #                         Py_ssize_t start,
@@ -410,6 +416,26 @@ def test_pop_byte(self):
         received = [j for j in l]
         self.assertEqual(received, expected)
 
+    def test_delitem(self):
+        l = List(self, 1, 0)
+        values = [b'a', b'b', b'c', b'd', b'e', b'f', b'g', b'h']
+        for i in values:
+            l.append(i)
+        self.assertEqual(len(l), 8)
+
+        # delete first item
+        del l[0]
+        self.assertEqual(len(l), 7)
+        self.assertEqual(list(l), values[1:])
+        # delete last item
+        del l[-1]
+        self.assertEqual(len(l), 6)
+        self.assertEqual(list(l), values[1:-1])
+        # delete item from middle
+        del l[2]
+        self.assertEqual(len(l), 5)
+        self.assertEqual(list(l), [b'b', b'c', b'e', b'f', b'g'])
+
     def test_delete_slice(self):
         l = List(self, 1, 0)
         values = [b'a', b'b', b'c', b'd', b'e', b'f', b'g', b'h']
diff --git a/numba/tests/test_typedlist.py b/numba/tests/test_typedlist.py
--- a/numba/tests/test_typedlist.py
+++ b/numba/tests/test_typedlist.py
@@ -619,6 +619,7 @@ def bar():
             return njit(context["bar"])
         for line1, line2 in (
                 ("lst.append(None)", "lst.pop()"),
+                ("lst.append(None)", "del lst[0]"),
                 ("lst.append(None)", "lst.count(None)"),
                 ("lst.append(None)", "lst.index(None)"),
                 ("lst.append(None)", "lst.insert(0, None)"),
@@ -1014,6 +1015,28 @@ def foo():
         # Need to compare the nested arrays
         self.assertTrue(np.all(expected[0] == got[0]))
 
+    def test_array_pop_from_single_value_list(self):
+        @njit
+        def foo():
+            l = List((np.zeros((1,)),))
+            l.pop()
+            return l
+
+        expected, got = foo.py_func(), foo()
+        # Need to compare the nested arrays
+        self.assertEqual(len(expected), 0)
+        self.assertEqual(len(got), 0)
+        # FIXME comparison of empty array-typed lists fails
+        # self.assertEqual(expected, got)
+
+    def test_5264(self):
+        # Test the reproducer from #5264 and make sure it doesn't segfault
+        float_array = types.float64[:]
+        l = List.empty_list(float_array)
+        l.append(np.ones(3,dtype=np.float64))
+        l.pop()
+        self.assertEqual(0, len(l))
+
     def test_jitclass_as_item_in_list(self):
 
         spec = [
@@ -1054,6 +1077,22 @@ def bag_equal(one, two):
 
         [bag_equal(a, b) for a, b in zip(expected, got)]
 
+    def test_4960(self):
+        # Test the reproducer from #4960 and make sure it doesn't segfault
+        @jitclass([('value', int32)])
+        class Simple(object):
+            def __init__(self, value):
+                self.value = value
+
+        @njit
+        def foo():
+            l = List((Simple(23),Simple(24)))
+            l.pop()
+            return l
+
+        l = foo()
+        self.assertEqual(1, len(l))
+
     def test_storage_model_mismatch(self):
         # https://github.com/numba/numba/issues/4520
         # check for storage model mismatch in refcount ops generation