Repository: numba/numba

Issue:
0.29.0 NotImplementedError, LoweringError with raise
``` py
import numba

def test0(n):
    return n

@numba.jit
def test1(n):
    if n == 0:
        raise ValueError()
    m = test0(n)
    s = 0
    for i in range(m):
        s += 1
    return s

test1(10)
```

This yields the following errors with 0.29.0, but not with 0.28.1 or 0.26.0. :

```
---------------------------------------------------------------------------
NotImplementedError                       Traceback (most recent call last)
/Applications/anaconda/envs/numba029/lib/python3.5/site-packages/numba/errors.py in new_error_context(fmt_, *args, **kwargs)
    242     try:
--> 243         yield
    244     except NumbaError as e:

/Applications/anaconda/envs/numba029/lib/python3.5/site-packages/numba/lowering.py in lower_block(self, block)
    200                                    loc=self.loc, errcls_=defaulterrcls):
--> 201                 self.lower_inst(inst)
    202 

/Applications/anaconda/envs/numba029/lib/python3.5/site-packages/numba/objmode.py in lower_inst(self, inst)
    144         else:
--> 145             raise NotImplementedError(type(inst), inst)
    146 

NotImplementedError: (<class 'numba.ir.StaticRaise'>, <numba.ir.StaticRaise object at 0x1102d10b8>)

During handling of the above exception, another exception occurred:

LoweringError                             Traceback (most recent call last)
<ipython-input-4-f2393023b331> in <module>()
----> 1 test1(10)

/Applications/anaconda/envs/numba029/lib/python3.5/site-packages/numba/dispatcher.py in _compile_for_args(self, *args, **kws)
    284                 argtypes.append(self.typeof_pyval(a))
    285         try:
--> 286             return self.compile(tuple(argtypes))
    287         except errors.TypingError as e:
    288             # Intercept typing error that may be due to an argument

/Applications/anaconda/envs/numba029/lib/python3.5/site-packages/numba/dispatcher.py in compile(self, sig)
    530 
    531             self._cache_misses[sig] += 1
--> 532             cres = self._compiler.compile(args, return_type)
    533             self.add_overload(cres)
    534             self._cache.save_overload(sig, cres)

/Applications/anaconda/envs/numba029/lib/python3.5/site-packages/numba/dispatcher.py in compile(self, args, return_type)
     79                                       impl,
     80                                       args=args, return_type=return_type,
---> 81                                       flags=flags, locals=self.locals)
     82         # Check typing error if object mode is used
     83         if cres.typing_error is not None and not flags.enable_pyobject:

/Applications/anaconda/envs/numba029/lib/python3.5/site-packages/numba/compiler.py in compile_extra(typingctx, targetctx, func, args, return_type, flags, locals, library)
    682     pipeline = Pipeline(typingctx, targetctx, library,
    683                         args, return_type, flags, locals)
--> 684     return pipeline.compile_extra(func)
    685 
    686 

/Applications/anaconda/envs/numba029/lib/python3.5/site-packages/numba/compiler.py in compile_extra(self, func)
    346         self.lifted = ()
    347         self.lifted_from = None
--> 348         return self._compile_bytecode()
    349 
    350     def compile_ir(self, func_ir, lifted=(), lifted_from=None):

/Applications/anaconda/envs/numba029/lib/python3.5/site-packages/numba/compiler.py in _compile_bytecode(self)
    647         """
    648         assert self.func_ir is None
--> 649         return self._compile_core()
    650 
    651     def _compile_ir(self):

/Applications/anaconda/envs/numba029/lib/python3.5/site-packages/numba/compiler.py in _compile_core(self)
    634 
    635         pm.finalize()
--> 636         res = pm.run(self.status)
    637         if res is not None:
    638             # Early pipeline completion

/Applications/anaconda/envs/numba029/lib/python3.5/site-packages/numba/compiler.py in run(self, status)
    233                     # No more fallback pipelines?
    234                     if is_final_pipeline:
--> 235                         raise patched_exception
    236                     # Go to next fallback pipeline
    237                     else:

/Applications/anaconda/envs/numba029/lib/python3.5/site-packages/numba/compiler.py in run(self, status)
    225                 try:
    226                     event(stage_name)
--> 227                     stage()
    228                 except _EarlyPipelineCompletion as e:
    229                     return e.result

/Applications/anaconda/envs/numba029/lib/python3.5/site-packages/numba/compiler.py in stage_objectmode_frontend(self)
    413         if self.flags.enable_looplift:
    414             assert not self.lifted
--> 415             cres = self.frontend_looplift()
    416             if cres is not None:
    417                 raise _EarlyPipelineCompletion(cres)

/Applications/anaconda/envs/numba029/lib/python3.5/site-packages/numba/compiler.py in frontend_looplift(self)
    404                               self.args, self.return_type,
    405                               outer_flags, self.locals,
--> 406                               lifted=tuple(loops), lifted_from=None)
    407             return cres
    408 

/Applications/anaconda/envs/numba029/lib/python3.5/site-packages/numba/compiler.py in compile_ir(typingctx, targetctx, func_ir, args, return_type, flags, locals, lifted, lifted_from, library)
    696                         args, return_type, flags, locals)
    697     return pipeline.compile_ir(func_ir=func_ir, lifted=lifted,
--> 698                                lifted_from=lifted_from)
    699 
    700 

/Applications/anaconda/envs/numba029/lib/python3.5/site-packages/numba/compiler.py in compile_ir(self, func_ir, lifted, lifted_from)
    354 
    355         self._set_and_check_ir(func_ir)
--> 356         return self._compile_ir()
    357 
    358     def stage_analyze_bytecode(self):

/Applications/anaconda/envs/numba029/lib/python3.5/site-packages/numba/compiler.py in _compile_ir(self)
    654         """
    655         assert self.func_ir is not None
--> 656         return self._compile_core()
    657 
    658 

/Applications/anaconda/envs/numba029/lib/python3.5/site-packages/numba/compiler.py in _compile_core(self)
    634 
    635         pm.finalize()
--> 636         res = pm.run(self.status)
    637         if res is not None:
    638             # Early pipeline completion

/Applications/anaconda/envs/numba029/lib/python3.5/site-packages/numba/compiler.py in run(self, status)
    233                     # No more fallback pipelines?
    234                     if is_final_pipeline:
--> 235                         raise patched_exception
    236                     # Go to next fallback pipeline
    237                     else:

/Applications/anaconda/envs/numba029/lib/python3.5/site-packages/numba/compiler.py in run(self, status)
    225                 try:
    226                     event(stage_name)
--> 227                     stage()
    228                 except _EarlyPipelineCompletion as e:
    229                     return e.result

/Applications/anaconda/envs/numba029/lib/python3.5/site-packages/numba/compiler.py in stage_objectmode_backend(self)
    552         """
    553         lowerfn = self.backend_object_mode
--> 554         self._backend(lowerfn, objectmode=True)
    555 
    556         # Warn if compiled function in object mode and force_pyobject not set

/Applications/anaconda/envs/numba029/lib/python3.5/site-packages/numba/compiler.py in _backend(self, lowerfn, objectmode)
    529             self.library.enable_object_caching()
    530 
--> 531         lowered = lowerfn()
    532         signature = typing.signature(self.return_type, *self.args)
    533         self.cr = compile_result(typing_context=self.typingctx,

/Applications/anaconda/envs/numba029/lib/python3.5/site-packages/numba/compiler.py in backend_object_mode(self)
    503                                      self.library,
    504                                      self.func_ir,
--> 505                                      self.flags)
    506 
    507     def backend_nopython_mode(self):

/Applications/anaconda/envs/numba029/lib/python3.5/site-packages/numba/compiler.py in py_lowering_stage(targetctx, library, interp, flags)
    824     fndesc = funcdesc.PythonFunctionDescriptor.from_object_mode_function(interp)
    825     lower = objmode.PyLower(targetctx, library, fndesc, interp)
--> 826     lower.lower()
    827     if not flags.no_cpython_wrapper:
    828         lower.create_cpython_wrapper()

/Applications/anaconda/envs/numba029/lib/python3.5/site-packages/numba/lowering.py in lower(self)
    124         if self.generator_info is None:
    125             self.genlower = None
--> 126             self.lower_normal_function(self.fndesc)
    127         else:
    128             self.genlower = self.GeneratorLower(self)

/Applications/anaconda/envs/numba029/lib/python3.5/site-packages/numba/lowering.py in lower_normal_function(self, fndesc)
    159         # Init argument values
    160         self.extract_function_arguments()
--> 161         entry_block_tail = self.lower_function_body()
    162 
    163         # Close tail of entry block

/Applications/anaconda/envs/numba029/lib/python3.5/site-packages/numba/lowering.py in lower_function_body(self)
    184             bb = self.blkmap[offset]
    185             self.builder.position_at_end(bb)
--> 186             self.lower_block(block)
    187 
    188         self.post_lower()

/Applications/anaconda/envs/numba029/lib/python3.5/site-packages/numba/lowering.py in lower_block(self, block)
    199             with new_error_context('lowering "{inst}" at {loc}', inst=inst,
    200                                    loc=self.loc, errcls_=defaulterrcls):
--> 201                 self.lower_inst(inst)
    202 
    203     def create_cpython_wrapper(self, release_gil=False):

/Applications/anaconda/envs/numba029/lib/python3.5/contextlib.py in __exit__(self, type, value, traceback)
     75                 value = type()
     76             try:
---> 77                 self.gen.throw(type, value, traceback)
     78                 raise RuntimeError("generator didn't stop after throw()")
     79             except StopIteration as exc:

/Applications/anaconda/envs/numba029/lib/python3.5/site-packages/numba/errors.py in new_error_context(fmt_, *args, **kwargs)
    247     except Exception as e:
    248         newerr = errcls(e).add_context(_format_msg(fmt_, args, kwargs))
--> 249         six.reraise(type(newerr), newerr, sys.exc_info()[2])
    250 
    251 

/Applications/anaconda/envs/numba029/lib/python3.5/site-packages/numba/six.py in reraise(tp, value, tb)
    656             value = tp()
    657         if value.__traceback__ is not tb:
--> 658             raise value.with_traceback(tb)
    659         raise value
    660 

/Applications/anaconda/envs/numba029/lib/python3.5/site-packages/numba/errors.py in new_error_context(fmt_, *args, **kwargs)
    241     errcls = kwargs.pop('errcls_', InternalError)
    242     try:
--> 243         yield
    244     except NumbaError as e:
    245         e.add_context(_format_msg(fmt_, args, kwargs))

/Applications/anaconda/envs/numba029/lib/python3.5/site-packages/numba/lowering.py in lower_block(self, block)
    199             with new_error_context('lowering "{inst}" at {loc}', inst=inst,
    200                                    loc=self.loc, errcls_=defaulterrcls):
--> 201                 self.lower_inst(inst)
    202 
    203     def create_cpython_wrapper(self, release_gil=False):

/Applications/anaconda/envs/numba029/lib/python3.5/site-packages/numba/objmode.py in lower_inst(self, inst)
    143 
    144         else:
--> 145             raise NotImplementedError(type(inst), inst)
    146 
    147     def lower_assign(self, inst):

LoweringError: Failed at object (object mode frontend)
Failed at object (object mode backend)
(<class 'numba.ir.StaticRaise'>, <numba.ir.StaticRaise object at 0x1102d10b8>)
File "<ipython-input-3-538140acc11f>", line 4
[1] During: lowering "raise <class 'ValueError'>()" at <ipython-input-3-538140acc11f> (4)
```



Discussion:
Thanks for the report, I can confirm. Seems like it first appeared in a83572e996487.

```
$ git bisect log
# bad: [9789438107948b73160569481926de14490a7e16] Fix typo in issue number
# good: [11bac3da6a489c52c3d6d06aac87d20026ddaf61] Merge pull request #1869 from sklam/fix/iss1868
git bisect start 'HEAD' 'HEAD~160'
# good: [19d5a49465cb864c7dc535cc1ccd604119a3a3e3] Refactor LAPACK/BLAS signatures.
git bisect good 19d5a49465cb864c7dc535cc1ccd604119a3a3e3
# good: [59bb1d830cd840d44056892e021f2e8b78c98457] Merge pull request #2060 from pitrou/bump_osx_deployment_target
git bisect good 59bb1d830cd840d44056892e021f2e8b78c98457
# bad: [cac5b00890761ae21ab22e0b3ec70a75a96378d5] Merge branch 'master' of github.com:numba/numba into refactor_interp
git bisect bad cac5b00890761ae21ab22e0b3ec70a75a96378d5
# good: [7593fd9a1aca0ab5cbb63a33315056338b183cfc] Merge branch 'master' into refactor_interp
git bisect good 7593fd9a1aca0ab5cbb63a33315056338b183cfc
# bad: [fe96f9950a404beb78e9989a0510c9719a06862a] Remove barely used properties
git bisect bad fe96f9950a404beb78e9989a0510c9719a06862a
# good: [a4e6d6689d11ddad4125a01c4e4ad19bc69c5759] Merge pull request #2119 from sklam/fix/iss2118
git bisect good a4e6d6689d11ddad4125a01c4e4ad19bc69c5759
# good: [43f2594ee953fbe0282865c64fa574b832a946f5] Refactor some bytecode handling
git bisect good 43f2594ee953fbe0282865c64fa574b832a946f5
# good: [e79230b1a92f33fa8db2f5aa5109924a75b145bb] s/interp/func_ir/
git bisect good e79230b1a92f33fa8db2f5aa5109924a75b145bb
# bad: [a83572e996487892bac533779a3d15c4ac2a00f4] Remove dual Interpreter constructors
git bisect bad a83572e996487892bac533779a3d15c4ac2a00f4
# first bad commit: [a83572e996487892bac533779a3d15c4ac2a00f4] Remove dual Interpreter constructors
```

I think this is because the raise is interpreted as a `StaticRaise` for which a handler for lowering the type is missing.


Resolving patch:
diff --git a/numba/compiler.py b/numba/compiler.py
--- a/numba/compiler.py
+++ b/numba/compiler.py
@@ -267,6 +267,7 @@ def __init__(self, typingctx, targetctx, library, args, return_type, flags,
         self.bc = None
         self.func_id = None
         self.func_ir = None
+        self.func_ir_original = None  # used for fallback
         self.lifted = None
         self.lifted_from = None
         self.typemap = None
@@ -377,6 +378,9 @@ def _set_and_check_ir(self, func_ir):
     def stage_process_ir(self):
         ir_processing_stage(self.func_ir)
 
+    def stage_preserve_ir(self):
+        self.func_ir_original = self.func_ir.copy()
+
     def frontend_looplift(self):
         """
         Loop lifting analysis and transformation
@@ -410,6 +414,7 @@ def stage_objectmode_frontend(self):
         """
         Front-end: Analyze bytecode, generate Numba IR, infer types
         """
+        self.func_ir = self.func_ir_original or self.func_ir
         if self.flags.enable_looplift:
             assert not self.lifted
             cres = self.frontend_looplift()
@@ -609,6 +614,8 @@ def _compile_core(self):
                 pm.add_stage(self.stage_analyze_bytecode, "analyzing bytecode")
             pm.add_stage(self.stage_process_ir, "processing IR")
             if not self.flags.no_rewrites:
+                if self.status.can_fallback:
+                    pm.add_stage(self.stage_preserve_ir, "preserve IR for fallback")
                 pm.add_stage(self.stage_generic_rewrites, "nopython rewrites")
             pm.add_stage(self.stage_nopython_frontend, "nopython frontend")
             pm.add_stage(self.stage_annotate_type, "annotate type")
diff --git a/numba/ir.py b/numba/ir.py
--- a/numba/ir.py
+++ b/numba/ir.py
@@ -845,6 +845,11 @@ def derive(self, blocks, arg_count=None, arg_names=None,
 
         return new_ir
 
+    def copy(self):
+        new_ir = copy.copy(self)
+        new_ir.blocks = self.blocks.copy()
+        return new_ir
+
     def get_block_entry_vars(self, block):
         """
         Return a set of variable names possibly alive at the beginning of
diff --git a/numba/rewrites/static_getitem.py b/numba/rewrites/static_getitem.py
--- a/numba/rewrites/static_getitem.py
+++ b/numba/rewrites/static_getitem.py
@@ -11,7 +11,7 @@ class RewriteConstGetitems(Rewrite):
     """
 
     def match(self, interp, block, typemap, calltypes):
-        self.getitems = getitems = []
+        self.getitems = getitems = {}
         self.block = block
         # Detect all getitem expressions and find which ones can be
         # rewritten
@@ -21,7 +21,7 @@ def match(self, interp, block, typemap, calltypes):
                     const = interp.infer_constant(expr.index)
                 except errors.ConstantInferenceError:
                     continue
-                getitems.append((expr, const))
+                getitems[expr] = const
 
         return len(getitems) > 0
 
@@ -29,11 +29,21 @@ def apply(self):
         """
         Rewrite all matching getitems as static_getitems.
         """
-        for expr, const in self.getitems:
-            expr.op = 'static_getitem'
-            expr.index_var = expr.index
-            expr.index = const
-        return self.block
+        new_block = self.block.copy()
+        new_block.clear()
+        for inst in self.block.body:
+            if isinstance(inst, ir.Assign):
+                expr = inst.value
+                if expr in self.getitems:
+                    const = self.getitems[expr]
+                    new_expr = ir.Expr.static_getitem(value=expr.value,
+                                                      index=const,
+                                                      index_var=expr.index,
+                                                      loc=expr.loc)
+                    inst = ir.Assign(value=new_expr, target=inst.target,
+                                     loc=inst.loc)
+            new_block.append(inst)
+        return new_block
 
 
 @register_rewrite('before-inference')


Test patch:
diff --git a/numba/tests/test_object_mode.py b/numba/tests/test_object_mode.py
--- a/numba/tests/test_object_mode.py
+++ b/numba/tests/test_object_mode.py
@@ -108,5 +108,56 @@ def test_delitem(self):
             cfunc(42)
 
 
+class TestObjectModeInvalidRewrite(TestCase):
+    """
+    Tests to ensure that rewrite passes didn't affect objmode lowering.
+    """
+
+    def _ensure_objmode(self, disp):
+        self.assertTrue(disp.signatures)
+        self.assertFalse(disp.nopython_signatures)
+        return disp
+
+    def test_static_raise_in_objmode_fallback(self):
+        """
+        Test code based on user submitted issue at
+        https://github.com/numba/numba/issues/2159
+        """
+        def test0(n):
+            return n
+
+        def test1(n):
+            if n == 0:
+                # static raise will fail in objmode if the IR is modified by
+                # rewrite pass
+                raise ValueError()
+            return test0(n)  # trigger objmode fallback
+
+        compiled = jit(test1)
+        self.assertEqual(test1(10), compiled(10))
+        self._ensure_objmode(compiled)
+
+    def test_static_setitem_in_objmode_fallback(self):
+        """
+        Test code based on user submitted issue at
+        https://github.com/numba/numba/issues/2169
+        """
+
+        def test0(n):
+            return n
+
+        def test(a1, a2):
+            a1 = np.asarray(a1)
+            # static setitem here will fail in objmode if the IR is modified by
+            # rewrite pass
+            a2[0] = 1
+            return test0(a1.sum() + a2.sum())   # trigger objmode fallback
+
+        compiled = jit(test)
+        args = np.array([3]), np.array([4])
+        self.assertEqual(test(*args), compiled(*args))
+        self._ensure_objmode(compiled)
+
+
 if __name__ == '__main__':
     unittest.main()