Repository: numba/numba

Issue:
RewriteArrayOfConsts incorrectly deleting `build_list` for empty list.
I ran this very simple code 
@jit(nopython=True)
def f(x):
	t = []
	myList = np.array([1,2,3])
	a = np.random.choice(myList, 2)
	t.append(x+a)
	return a

And I got this error, with request to report it here:

---------------------------------------------------------------------------
KeyError                                  Traceback (most recent call last)
~/anaconda3/envs/mne/lib/python3.6/site-packages/numba/errors.py in new_error_context(fmt_, *args, **kwargs)
    616     try:
--> 617         yield
    618     except NumbaError as e:

~/anaconda3/envs/mne/lib/python3.6/site-packages/numba/lowering.py in lower_block(self, block)
    258                                    loc=self.loc, errcls_=defaulterrcls):
--> 259                 self.lower_inst(inst)
    260 

~/anaconda3/envs/mne/lib/python3.6/site-packages/numba/lowering.py in lower_inst(self, inst)
    307             ty = self.typeof(inst.target.name)
--> 308             val = self.lower_assign(ty, inst)
    309             self.storevar(val, inst.target.name)

~/anaconda3/envs/mne/lib/python3.6/site-packages/numba/lowering.py in lower_assign(self, ty, inst)
    456         elif isinstance(value, ir.Var):
--> 457             val = self.loadvar(value.name)
    458             oty = self.typeof(value.name)

~/anaconda3/envs/mne/lib/python3.6/site-packages/numba/lowering.py in loadvar(self, name)
   1070         """
-> 1071         ptr = self.getvar(name)
   1072         return self.builder.load(ptr)

~/anaconda3/envs/mne/lib/python3.6/site-packages/numba/lowering.py in getvar(self, name)
   1064         """
-> 1065         return self.varmap[name]
   1066 

KeyError: '$0.1'

During handling of the above exception, another exception occurred:

LoweringError                             Traceback (most recent call last)
~/my_code/genetic_algorithm/fnc_try_parall.py in <module>
     19 
     20 
---> 21 b = f(2)
     22 print(b)
     23 

~/anaconda3/envs/mne/lib/python3.6/site-packages/numba/dispatcher.py in _compile_for_args(self, *args, **kws)
    365                     e.patch_message(''.join(e.args) + help_msg)
    366             # ignore the FULL_TRACEBACKS config, this needs reporting!
--> 367             raise e
    368 
    369     def inspect_llvm(self, signature=None):

~/anaconda3/envs/mne/lib/python3.6/site-packages/numba/dispatcher.py in _compile_for_args(self, *args, **kws)
    322                 argtypes.append(self.typeof_pyval(a))
    323         try:
--> 324             return self.compile(tuple(argtypes))
    325         except errors.TypingError as e:
    326             # Intercept typing error that may be due to an argument

~/anaconda3/envs/mne/lib/python3.6/site-packages/numba/compiler_lock.py in _acquire_compile_lock(*args, **kwargs)
     30         def _acquire_compile_lock(*args, **kwargs):
     31             with self:
---> 32                 return func(*args, **kwargs)
     33         return _acquire_compile_lock
     34 

~/anaconda3/envs/mne/lib/python3.6/site-packages/numba/dispatcher.py in compile(self, sig)
    653 
    654             self._cache_misses[sig] += 1
--> 655             cres = self._compiler.compile(args, return_type)
    656             self.add_overload(cres)
    657             self._cache.save_overload(sig, cres)

~/anaconda3/envs/mne/lib/python3.6/site-packages/numba/dispatcher.py in compile(self, args, return_type)
     80                                       args=args, return_type=return_type,
     81                                       flags=flags, locals=self.locals,
---> 82                                       pipeline_class=self.pipeline_class)
     83         # Check typing error if object mode is used
     84         if cres.typing_error is not None and not flags.enable_pyobject:

~/anaconda3/envs/mne/lib/python3.6/site-packages/numba/compiler.py in compile_extra(typingctx, targetctx, func, args, return_type, flags, locals, library, pipeline_class)
    924     pipeline = pipeline_class(typingctx, targetctx, library,
    925                               args, return_type, flags, locals)
--> 926     return pipeline.compile_extra(func)
    927 
    928 

~/anaconda3/envs/mne/lib/python3.6/site-packages/numba/compiler.py in compile_extra(self, func)
    372         self.lifted = ()
    373         self.lifted_from = None
--> 374         return self._compile_bytecode()
    375 
    376     def compile_ir(self, func_ir, lifted=(), lifted_from=None):

~/anaconda3/envs/mne/lib/python3.6/site-packages/numba/compiler.py in _compile_bytecode(self)
    855         """
    856         assert self.func_ir is None
--> 857         return self._compile_core()
    858 
    859     def _compile_ir(self):

~/anaconda3/envs/mne/lib/python3.6/site-packages/numba/compiler.py in _compile_core(self)
    842         self.define_pipelines(pm)
    843         pm.finalize()
--> 844         res = pm.run(self.status)
    845         if res is not None:
    846             # Early pipeline completion

~/anaconda3/envs/mne/lib/python3.6/site-packages/numba/compiler_lock.py in _acquire_compile_lock(*args, **kwargs)
     30         def _acquire_compile_lock(*args, **kwargs):
     31             with self:
---> 32                 return func(*args, **kwargs)
     33         return _acquire_compile_lock
     34 

~/anaconda3/envs/mne/lib/python3.6/site-packages/numba/compiler.py in run(self, status)
    253                     # No more fallback pipelines?
    254                     if is_final_pipeline:
--> 255                         raise patched_exception
    256                     # Go to next fallback pipeline
    257                     else:

~/anaconda3/envs/mne/lib/python3.6/site-packages/numba/compiler.py in run(self, status)
    244                 try:
    245                     event(stage_name)
--> 246                     stage()
    247                 except _EarlyPipelineCompletion as e:
    248                     return e.result

~/anaconda3/envs/mne/lib/python3.6/site-packages/numba/compiler.py in stage_nopython_backend(self)
    715         """
    716         lowerfn = self.backend_nopython_mode
--> 717         self._backend(lowerfn, objectmode=False)
    718 
    719     def stage_compile_interp_mode(self):

~/anaconda3/envs/mne/lib/python3.6/site-packages/numba/compiler.py in _backend(self, lowerfn, objectmode)
    664             self.library.enable_object_caching()
    665 
--> 666         lowered = lowerfn()
    667         signature = typing.signature(self.return_type, *self.args)
    668         self.cr = compile_result(

~/anaconda3/envs/mne/lib/python3.6/site-packages/numba/compiler.py in backend_nopython_mode(self)
    651                 self.calltypes,
    652                 self.flags,
--> 653                 self.metadata)
    654 
    655     def _backend(self, lowerfn, objectmode):

~/anaconda3/envs/mne/lib/python3.6/site-packages/numba/compiler.py in native_lowering_stage(targetctx, library, interp, typemap, restype, calltypes, flags, metadata)
   1046         lower = lowering.Lower(targetctx, library, fndesc, interp,
   1047                                metadata=metadata)
-> 1048         lower.lower()
   1049         if not flags.no_cpython_wrapper:
   1050             lower.create_cpython_wrapper(flags.release_gil)

~/anaconda3/envs/mne/lib/python3.6/site-packages/numba/lowering.py in lower(self)
    176         if self.generator_info is None:
    177             self.genlower = None
--> 178             self.lower_normal_function(self.fndesc)
    179         else:
    180             self.genlower = self.GeneratorLower(self)

~/anaconda3/envs/mne/lib/python3.6/site-packages/numba/lowering.py in lower_normal_function(self, fndesc)
    217         # Init argument values
    218         self.extract_function_arguments()
--> 219         entry_block_tail = self.lower_function_body()
    220 
    221         # Close tail of entry block

~/anaconda3/envs/mne/lib/python3.6/site-packages/numba/lowering.py in lower_function_body(self)
    242             bb = self.blkmap[offset]
    243             self.builder.position_at_end(bb)
--> 244             self.lower_block(block)
    245 
    246         self.post_lower()

~/anaconda3/envs/mne/lib/python3.6/site-packages/numba/lowering.py in lower_block(self, block)
    257             with new_error_context('lowering "{inst}" at {loc}', inst=inst,
    258                                    loc=self.loc, errcls_=defaulterrcls):
--> 259                 self.lower_inst(inst)
    260 
    261     def create_cpython_wrapper(self, release_gil=False):

~/anaconda3/envs/mne/lib/python3.6/contextlib.py in __exit__(self, type, value, traceback)
     97                 value = type()
     98             try:
---> 99                 self.gen.throw(type, value, traceback)
    100             except StopIteration as exc:
    101                 # Suppress StopIteration *unless* it's the same exception that

~/anaconda3/envs/mne/lib/python3.6/site-packages/numba/errors.py in new_error_context(fmt_, *args, **kwargs)
    623         from numba import config
    624         tb = sys.exc_info()[2] if config.FULL_TRACEBACKS else None
--> 625         six.reraise(type(newerr), newerr, tb)
    626 
    627 

~/anaconda3/envs/mne/lib/python3.6/site-packages/numba/six.py in reraise(tp, value, tb)
    657         if value.__traceback__ is not tb:
    658             raise value.with_traceback(tb)
--> 659         raise value
    660 
    661 else:

LoweringError: Failed in nopython mode pipeline (step: nopython mode backend)
'$0.1'

File "fnc_try_parall.py", line 14:
def f(x):
	t = []
 ^

[1] During: lowering "t = $0.1" at /home/karin/my_code/genetic_algorithm/fnc_try_parall.py (14)
-------------------------------------------------------------------------------
This should not have happened, a problem has occurred in Numba's internals.

Please report the error message and traceback, along with a minimal reproducer
at: https://github.com/numba/numba/issues/new

If more help is needed please feel free to speak to the Numba core developers
directly at: https://gitter.im/numba/numba

Thanks in advance for your help in improving Numba!

Best, 
Karin


Discussion:
Thanks for the report, I can reproduce. Seems like the `RewriteArrayOfConsts` rewrite pass is incorrectly thinking that the list assigned to variable `t` is dead and so is removing it. I think I have a fix.

Resolving patch:
diff --git a/numba/inline_closurecall.py b/numba/inline_closurecall.py
--- a/numba/inline_closurecall.py
+++ b/numba/inline_closurecall.py
@@ -1026,76 +1026,102 @@ def inline_array(array_var, expr, stmts, list_vars, dels):
         stmts.extend(dels)
         return True
 
-    # list_vars keep track of the variable created from the latest
-    # build_list instruction, as well as its synonyms.
-    list_vars = []
-    # dead_vars keep track of those in list_vars that are considered dead.
-    dead_vars = []
-    # list_items keep track of the elements used in build_list.
-    list_items = []
-    stmts = []
-    # dels keep track of the deletion of list_items, which will need to be
-    # moved after array initialization.
-    dels = []
-    modified = False
+    class State(object):
+        """
+        This class is used to hold the state in the following loop so as to make
+        it easy to reset the state of the variables tracking the various
+        statement kinds
+        """
+
+        def __init__(self):
+            # list_vars keep track of the variable created from the latest
+            # build_list instruction, as well as its synonyms.
+            self.list_vars = []
+            # dead_vars keep track of those in list_vars that are considered dead.
+            self.dead_vars = []
+            # list_items keep track of the elements used in build_list.
+            self.list_items = []
+            self.stmts = []
+            # dels keep track of the deletion of list_items, which will need to be
+            # moved after array initialization.
+            self.dels = []
+            # tracks if a modification has taken place
+            self.modified = False
+
+        def reset(self):
+            """
+            Resets the internal state of the variables used for tracking
+            """
+            self.list_vars = []
+            self.dead_vars = []
+            self.list_items = []
+            self.dels = []
+
+        def list_var_used(self, inst):
+            """
+            Returns True if the list being analysed is used between the
+            build_list and the array call.
+            """
+            return any([x.name in self.list_vars for x in inst.list_vars()])
+
+    state = State()
+
     for inst in block.body:
         if isinstance(inst, ir.Assign):
             if isinstance(inst.value, ir.Var):
-                if inst.value.name in list_vars:
-                    list_vars.append(inst.target.name)
-                    stmts.append(inst)
+                if inst.value.name in state.list_vars:
+                    state.list_vars.append(inst.target.name)
+                    state.stmts.append(inst)
                     continue
             elif isinstance(inst.value, ir.Expr):
                 expr = inst.value
                 if expr.op == 'build_list':
-                    list_vars = [inst.target.name]
-                    list_items = [x.name for x in expr.items]
-                    stmts.append(inst)
+                    # new build_list encountered, reset state
+                    state.reset()
+                    state.list_items = [x.name for x in expr.items]
+                    state.list_vars = [inst.target.name]
+                    state.stmts.append(inst)
                     continue
                 elif expr.op == 'call' and expr in calltypes:
                     arr_var = inst.target
                     if guard(inline_array, inst.target, expr,
-                                           stmts, list_vars, dels):
-                        modified = True
+                             state.stmts, state.list_vars, state.dels):
+                        state.modified = True
                         continue
         elif isinstance(inst, ir.Del):
             removed_var = inst.value
-            if removed_var in list_items:
-                dels.append(inst)
+            if removed_var in state.list_items:
+                state.dels.append(inst)
                 continue
-            elif removed_var in list_vars:
+            elif removed_var in state.list_vars:
                 # one of the list_vars is considered dead.
-                dead_vars.append(removed_var)
-                list_vars.remove(removed_var)
-                stmts.append(inst)
-                if list_vars == []:
+                state.dead_vars.append(removed_var)
+                state.list_vars.remove(removed_var)
+                state.stmts.append(inst)
+                if state.list_vars == []:
                     # if all list_vars are considered dead, we need to filter
                     # them out from existing stmts to completely remove
                     # build_list.
                     # Note that if a translation didn't take place, dead_vars
                     # will also be empty when we reach this point.
                     body = []
-                    for inst in stmts:
+                    for inst in state.stmts:
                         if ((isinstance(inst, ir.Assign) and
-                             inst.target.name in dead_vars) or
+                             inst.target.name in state.dead_vars) or
                              (isinstance(inst, ir.Del) and
-                             inst.value in dead_vars)):
+                             inst.value in state.dead_vars)):
                             continue
                         body.append(inst)
-                    stmts = body
-                    dead_vars = []
-                    modified = True
+                    state.stmts = body
+                    state.dead_vars = []
+                    state.modified = True
                     continue
-        stmts.append(inst)
+        state.stmts.append(inst)
 
         # If the list is used in any capacity between build_list and array
         # call, then we must call off the translation for this list because
         # it could be mutated and list_items would no longer be applicable.
-        list_var_used = any([ x.name in list_vars for x in inst.list_vars() ])
-        if list_var_used:
-            list_vars = []
-            dead_vars = []
-            list_items = []
-            dels = []
-
-    return stmts if modified else None
+        if state.list_var_used(inst):
+            state.reset()
+
+    return state.stmts if state.modified else None


Test patch:
diff --git a/numba/tests/test_comprehension.py b/numba/tests/test_comprehension.py
--- a/numba/tests/test_comprehension.py
+++ b/numba/tests/test_comprehension.py
@@ -248,9 +248,9 @@ class TestArrayComprehension(unittest.TestCase):
     def check(self, pyfunc, *args, **kwargs):
         """A generic check function that run both pyfunc, and jitted pyfunc,
         and compare results."""
-        run_parallel = kwargs['run_parallel'] if 'run_parallel' in kwargs else False
-        assert_allocate_list = kwargs['assert_allocate_list'] if 'assert_allocate_list' in kwargs else False
-        assert_dtype = kwargs['assert_dtype'] if 'assert_dtype' in kwargs else None
+        run_parallel = kwargs.get('run_parallel', False)
+        assert_allocate_list = kwargs.get('assert_allocate_list', False)
+        assert_dtype = kwargs.get('assert_dtype', False)
         cfunc = jit(nopython=True,parallel=run_parallel)(pyfunc)
         pyres = pyfunc(*args)
         cres = cfunc(*args)
@@ -483,6 +483,15 @@ def foo():
         self.assertNotEqual(got, expect)
         self.assertRegexpMatches(got, r'\[(\s*\d+)+\]')
 
+    def test_empty_list_not_removed(self):
+        # see issue #3724
+        def f(x):
+            t = []
+            myList = np.array([1])
+            a = np.random.choice(myList, 1)
+            t.append(x + a)
+            return a
+        self.check(f, 5, assert_allocate_list=True)
 
 if __name__ == '__main__':
     unittest.main()