Repository: numba/numba

Issue:
Closure recursively calling itself fails in bytecode analysis
In testing https://github.com/numba/numba/pull/2369
Something like:
```python
from numba import njit

def outer(x):
    y = x + 1
    def inner(x):
        if x + y < 2:
            return 10
        else:
            inner(x - 1)
    return inner(x)

njit(outer)
```
Fails with a backtrace like:
```python
Traceback (most recent call last):
  File "<path>numba/numba/ir.py", line 69, in get
    return self._con[name]
KeyError: 'inner'

During handling of the above exception, another exception occurred:

Traceback (most recent call last):
  File "<path>numba/numba/ir.py", line 670, in get_exact
    return self.localvars.get(name)
  File "<path>numba/numba/ir.py", line 71, in get
    raise NotDefinedError(name)
numba.errors.NotDefinedError: 'inner' is not defined in None

During handling of the above exception, another exception occurred:

Traceback (most recent call last):
  File "<path>numba/numba/ir.py", line 69, in get
    return self._con[name]
KeyError: 'inner'

During handling of the above exception, another exception occurred:

Traceback (most recent call last):
  File "<path>numba/numba/tests/test_closure.py", line 218, in test_recursive_closure
    self.assertEqual(cfunc(var), ref(var))
  File "<path>numba/numba/dispatcher.py", line 307, in _compile_for_args
    return self.compile(tuple(argtypes))
  File "<path>numba/numba/dispatcher.py", line 579, in compile
    cres = self._compiler.compile(args, return_type)
  File "<path>numba/numba/dispatcher.py", line 80, in compile
    flags=flags, locals=self.locals)
  File "<path>numba/numba/compiler.py", line 740, in compile_extra
    return pipeline.compile_extra(func)
  File "<path>numba/numba/compiler.py", line 360, in compile_extra
    return self._compile_bytecode()
  File "<path>numba/numba/compiler.py", line 699, in _compile_bytecode
    return self._compile_core()
  File "<path>numba/numba/compiler.py", line 686, in _compile_core
    res = pm.run(self.status)
  File "<path>numba/numba/compiler.py", line 246, in run
    raise patched_exception
  File "<path>numba/numba/compiler.py", line 238, in run
    stage()
  File "<path>numba/numba/compiler.py", line 374, in stage_analyze_bytecode
    func_ir = translate_stage(self.func_id, self.bc)
  File "<path>numba/numba/compiler.py", line 803, in translate_stage
    return interp.interpret(bytecode)
  File "<path>numba/numba/interpreter.py", line 109, in interpret
    self._dispatch(inst, kws)
  File "<path>numba/numba/interpreter.py", line 238, in _dispatch
    raise e
  File "<path>numba/numba/interpreter.py", line 234, in _dispatch
    return fn(inst, **kws)
  File "<path>numba/numba/interpreter.py", line 971, in op_LOAD_CLOSURE
    gl = self.get(name)
  File "<path>numba/numba/interpreter.py", line 272, in get
    var = self.current_scope.get(name)
  File "<path>numba/numba/ir.py", line 662, in get
    return self.get_exact(name)
  File "<path>numba/numba/ir.py", line 673, in get_exact
    return self.parent.get(name)
  File "<path>numba/numba/ir.py", line 662, in get
    return self.get_exact(name)
  File "<path>numba/numba/ir.py", line 670, in get_exact
    return self.localvars.get(name)
  File "<path>numba/numba/ir.py", line 71, in get
    raise NotDefinedError(name)
numba.errors.NotDefinedError: 'inner' is not defined in <path>numba/numba/tests/test_closure.py (207)

```
The `Scope` class instance `localvars` attr is an empty dict so lookup fails and the `has_parent` is `False` so it cannot be queried.
Closure recursively calling itself fails in bytecode analysis
In testing https://github.com/numba/numba/pull/2369
Something like:
```python
from numba import njit

def outer(x):
    y = x + 1
    def inner(x):
        if x + y < 2:
            return 10
        else:
            inner(x - 1)
    return inner(x)

njit(outer)
```
Fails with a backtrace like:
```python
Traceback (most recent call last):
  File "<path>numba/numba/ir.py", line 69, in get
    return self._con[name]
KeyError: 'inner'

During handling of the above exception, another exception occurred:

Traceback (most recent call last):
  File "<path>numba/numba/ir.py", line 670, in get_exact
    return self.localvars.get(name)
  File "<path>numba/numba/ir.py", line 71, in get
    raise NotDefinedError(name)
numba.errors.NotDefinedError: 'inner' is not defined in None

During handling of the above exception, another exception occurred:

Traceback (most recent call last):
  File "<path>numba/numba/ir.py", line 69, in get
    return self._con[name]
KeyError: 'inner'

During handling of the above exception, another exception occurred:

Traceback (most recent call last):
  File "<path>numba/numba/tests/test_closure.py", line 218, in test_recursive_closure
    self.assertEqual(cfunc(var), ref(var))
  File "<path>numba/numba/dispatcher.py", line 307, in _compile_for_args
    return self.compile(tuple(argtypes))
  File "<path>numba/numba/dispatcher.py", line 579, in compile
    cres = self._compiler.compile(args, return_type)
  File "<path>numba/numba/dispatcher.py", line 80, in compile
    flags=flags, locals=self.locals)
  File "<path>numba/numba/compiler.py", line 740, in compile_extra
    return pipeline.compile_extra(func)
  File "<path>numba/numba/compiler.py", line 360, in compile_extra
    return self._compile_bytecode()
  File "<path>numba/numba/compiler.py", line 699, in _compile_bytecode
    return self._compile_core()
  File "<path>numba/numba/compiler.py", line 686, in _compile_core
    res = pm.run(self.status)
  File "<path>numba/numba/compiler.py", line 246, in run
    raise patched_exception
  File "<path>numba/numba/compiler.py", line 238, in run
    stage()
  File "<path>numba/numba/compiler.py", line 374, in stage_analyze_bytecode
    func_ir = translate_stage(self.func_id, self.bc)
  File "<path>numba/numba/compiler.py", line 803, in translate_stage
    return interp.interpret(bytecode)
  File "<path>numba/numba/interpreter.py", line 109, in interpret
    self._dispatch(inst, kws)
  File "<path>numba/numba/interpreter.py", line 238, in _dispatch
    raise e
  File "<path>numba/numba/interpreter.py", line 234, in _dispatch
    return fn(inst, **kws)
  File "<path>numba/numba/interpreter.py", line 971, in op_LOAD_CLOSURE
    gl = self.get(name)
  File "<path>numba/numba/interpreter.py", line 272, in get
    var = self.current_scope.get(name)
  File "<path>numba/numba/ir.py", line 662, in get
    return self.get_exact(name)
  File "<path>numba/numba/ir.py", line 673, in get_exact
    return self.parent.get(name)
  File "<path>numba/numba/ir.py", line 662, in get
    return self.get_exact(name)
  File "<path>numba/numba/ir.py", line 670, in get_exact
    return self.localvars.get(name)
  File "<path>numba/numba/ir.py", line 71, in get
    raise NotDefinedError(name)
numba.errors.NotDefinedError: 'inner' is not defined in <path>numba/numba/tests/test_closure.py (207)

```
The `Scope` class instance `localvars` attr is an empty dict so lookup fails and the `has_parent` is `False` so it cannot be queried.


Discussion:
Ideally, we would like to return a more understandable error indicating that recursive calls in closures are not allowed at this time.
Agreed. This sort of thing is fine, recursive function call in a closure...

```python
@njit
def fib3(n):
    if n < 2:
        return n
    return fib3(n - 1) + fib3(n - 2)

def outer(x):
    z = x + 1
    def inner(x):
        return x + fib3(z)
    return inner(x)

cfunc = njit(outer)
```
#2369 does not give full closure support. It only supports closure calls that can be fully inlined, so a closure function calling itself is not going to work here. I'll see if we can give better error messages in this case as @seibert suggested.
Closing. Handled in https://github.com/numba/numba/pull/2369.
Ideally, we would like to return a more understandable error indicating that recursive calls in closures are not allowed at this time.
Agreed. This sort of thing is fine, recursive function call in a closure...

```python
@njit
def fib3(n):
    if n < 2:
        return n
    return fib3(n - 1) + fib3(n - 2)

def outer(x):
    z = x + 1
    def inner(x):
        return x + fib3(z)
    return inner(x)

cfunc = njit(outer)
```
#2369 does not give full closure support. It only supports closure calls that can be fully inlined, so a closure function calling itself is not going to work here. I'll see if we can give better error messages in this case as @seibert suggested.
Closing. Handled in https://github.com/numba/numba/pull/2369.

Resolving patch:
diff --git a/numba/transforms.py b/numba/transforms.py
--- a/numba/transforms.py
+++ b/numba/transforms.py
@@ -9,6 +9,7 @@
 from numba.analysis import compute_cfg_from_blocks, find_top_level_loops
 from numba import ir
 from numba.interpreter import Interpreter
+from numba.analysis import compute_use_defs
 
 
 def _extract_loop_lifting_candidates(cfg, blocks):
@@ -61,12 +62,29 @@ def _loop_lift_get_candidate_infos(cfg, blocks, livemap):
         [callfrom] = loop.entries   # requirement checked earlier
         an_exit = next(iter(loop.exits))  # anyone of the exit block
         [(returnto, _)] = cfg.successors(an_exit)  # requirement checked earlier
+        inputs = livemap[callfrom]
+        outputs = livemap[returnto]
+
+        # ensure live variables are actually used in the blocks, else remove,
+        # saves having to create something valid to run through postproc
+        # to achieve similar
+        local_block_ids = set(loop.body) | set(loop.entries) | set(loop.exits)
+        loopblocks = {}
+        for k in local_block_ids:
+            loopblocks[k] = blocks[k]
+
+        used_vars = set()
+        for vs in compute_use_defs(loopblocks).usemap.values():
+            used_vars |= vs
+
         # note: sorted for stable ordering
-        inputs = sorted(livemap[callfrom])
-        outputs = sorted(livemap[returnto])
+        inputs = sorted(set(inputs) & used_vars)
+        outputs = sorted(set(outputs) & used_vars)
+
         lli = _loop_lift_info(loop=loop, inputs=inputs, outputs=outputs,
                               callfrom=callfrom, returnto=returnto)
         loopinfos.append(lli)
+
     return loopinfos
 
 


Test patch:
diff --git a/numba/tests/test_looplifting.py b/numba/tests/test_looplifting.py
--- a/numba/tests/test_looplifting.py
+++ b/numba/tests/test_looplifting.py
@@ -343,6 +343,31 @@ def test(x):
         x = np.array([1., 4, 2, -3, 5, 2, 10, 5, 2, 6])
         np.testing.assert_equal(test.py_func(x), test(x))
 
+    def test_issue_2368(self):
+        from numba import jit
+
+        def lift_issue2368(a, b):
+            s = 0
+            for e in a:
+                s += e
+            h = b.__hash__()
+            return s, h
+
+        a = np.ones(10)
+        b = object()
+        jitted = jit(lift_issue2368)
+
+        expected = lift_issue2368(a, b)
+        got = jitted(a, b)
+
+        self.assertEqual(expected[0], got[0])
+        self.assertEqual(expected[1], got[1])
+
+        jitloop = jitted.overloads[jitted.signatures[0]].lifted[0]
+        [loopcres] = jitloop.overloads.values()
+        # assert lifted function is native
+        self.assertTrue(loopcres.fndesc.native)
+
     def test_no_iteration(self):
         from numba import jit