Repository: numba/numba

Issue:
"Fatal Python error: deallocating None" at run time

Using latest Numba, 0.43.1 with Python 3.7.1 on Xubuntu 18.04.
A minimal reproducer is pasted below.

Under some conditions, a jit'd function will reduce the reference count for "None".
See reproducer code for more information.

```
import os
import sys

print('-- py:', sys.version.replace('\n', ' '))
from numba import jit
import numba
print('-- numba:', numba.__version__)


# Monitor None's reference count

HI_NRC = LO_NRC = NRC = sys.getrefcount(None)
def checkNoneRefCnt():
    global HI_NRC, LO_NRC, NRC
    z = sys.getrefcount(None)
    if z > HI_NRC:
        HI_NRC = z
    if z < LO_NRC:
        print('!! {:6d} -> {:6d}  ({:d})'.format(LO_NRC, z, z - LO_NRC))
        LO_NRC = z
    NRC = z


# Function under test:
#   Returns first & last indices of positive xyz values, and a count of them.
#   If "if a is None" is not executed, None's reference count shrinks.
#   Shrinkage is usually, but not always, 2 when running in vscode.

@jit(nopython=True)
def foo(lo, hi, xyz):
    fx, lx, n = None, None, 0
    for x in range(lo, hi + 1):
        if xyz[x] > 0:
            if fx is None:
                fx = x
            lx = x
            n += 1
    return fx, lx, n


def main():
    xyz = [-1, 0, 1]    # The only positive value is in the last cell.
    #lox, hix = 0, 2     # OK when last cell is seen.
    lox, hix = 0, 1     # Fails when last cell is not seen.
    for _ in range(999999):
        _ = foo(lox, hix, xyz)
        checkNoneRefCnt()

if __name__ == '__main__':
    main()
```
Thank you



Discussion:
Thanks for the report, I can reproduce. The MWE can be reduced to:
```python
import os
import sys

print('-- py:', sys.version.replace('\n', ' '))
from numba import jit
import numba
print('-- numba:', numba.__version__)


# Monitor None's reference count

HI_NRC = LO_NRC = NRC = sys.getrefcount(None)
def checkNoneRefCnt():
    global HI_NRC, LO_NRC, NRC
    z = sys.getrefcount(None)
    if z > HI_NRC:
        HI_NRC = z
    if z < LO_NRC:
        print('!! {:6d} -> {:6d}  ({:d})'.format(LO_NRC, z, z - LO_NRC))
        LO_NRC = z
    NRC = z

@jit(nopython=True)
def foo(maybe):
    lx = None
    if maybe:
        lx = 10
    return 1, lx

def main():
    tmp = []
    for _ in range(20000):
        maybe = False
        _ = foo(maybe)
        checkNoneRefCnt()

if __name__ == '__main__':
    main()
```
and run with (for a consistent deallocation behaviour):
```
PYTHONHASHSEED=0 python issue4058.py
```
I suspect there's a decref issue surrounding optional return types in a tuple. If the predicate `maybe` is adjusted such that it is `True` more often the problem is less prevalent. I think the root cause may be here:

https://github.com/numba/numba/blob/1de3cd5a687011d90b1a26070c6fe47abdf8a2a5/numba/targets/boxing.py#L260-L265

and that if an Optional it not valid as a value then `None` needs an incref.
Thank you very much for the speedy response. 

On top of being super useful, the numba project appears to be in good hands!
@kdresser no problem, thanks for your kind words and for using Numba :)
@sklam any view on the above? If you agree to the cause I'm happy to patch, else we can discuss! Thanks!
@stuartarchibald 

> if an Optional it not valid as a value then None needs an incref.

You are right.  It just need a incref for the `None`

Resolving patch:
diff --git a/numba/targets/boxing.py b/numba/targets/boxing.py
--- a/numba/targets/boxing.py
+++ b/numba/targets/boxing.py
@@ -253,14 +253,16 @@ def unbox_charseq(typ, obj, c):
     return NativeValue(ret, is_error=c.builder.not_(ok))
 
 
-
 @box(types.Optional)
 def box_optional(typ, val, c):
     optval = c.context.make_helper(c.builder, typ, val)
     ret = cgutils.alloca_once_value(c.builder, c.pyapi.borrow_none())
-    with c.builder.if_then(optval.valid):
-        validres = c.box(typ.type, optval.data)
-        c.builder.store(validres, ret)
+    with c.builder.if_else(optval.valid) as (then, otherwise):
+        with then:
+            validres = c.box(typ.type, optval.data)
+            c.builder.store(validres, ret)
+        with otherwise:
+            c.builder.store(c.pyapi.make_none(), ret)
     return c.builder.load(ret)
 
 


Test patch:
diff --git a/numba/tests/test_optional.py b/numba/tests/test_optional.py
--- a/numba/tests/test_optional.py
+++ b/numba/tests/test_optional.py
@@ -240,6 +240,27 @@ def pyfunc(x):
         self.assertEqual(pyfunc(None), cfunc(None))
         self.assertEqual(pyfunc((1, 2)), cfunc((1, 2)))
 
+    def test_many_optional_none_returns(self):
+        """
+        Issue #4058
+        """
+        @njit
+        def foo(maybe):
+            lx = None
+            if maybe:
+                lx = 10
+            return 1, lx
+
+        def work():
+            tmp = []
+            for _ in range(20000):
+                maybe = False
+                _ = foo(maybe)
+
+        # this caused "Fatal Python error: deallocating None" as there was no
+        # incref being made on the returned None.
+        work()
+
 
 if __name__ == '__main__':
     unittest.main()