Repository: numba/numba

Issue:
Failure in step: convert to parfors
- [X] I am using the latest released version of Numba (most recent is visible in
 the change log (https://github.com/numba/numba/blob/master/CHANGE_LOG).
- [X] I have included below a minimal working reproducer (if you are unsure how
 to write one see http://matthewrocklin.com/blog/work/2018/02/28/minimal-bug-reports).

Code which used to compile successfully using older versions of Numba/llvmlite (verified on 0.43.1/v0.28.0) now raises the following error:

```
TypeError: Failed in nopython mode pipeline (step: convert to parfors)
'NoneType' object is not iterable
```

I have set up the following MWR:
```python
import numpy as np
from numba import jit, prange


@jit(nopython=True, fastmath=True, parallel=True, cache=True, nogil=True)
def reproducer(arr):

    n_valid_vals = 0

    for i in prange(arr.shape[0]):

        if arr[i] != 0:
            n_valid_vals += 1

        if n_valid_vals:
            print(arr[:n_valid_vals])


if __name__ == "__main__":
    arr = np.random.random((3,))
    reproducer(arr)

```


Discussion:
Bug confirmed; not 0.49 regression 

<details>
<summary>traceback on 0.48.0</summary>

```pytb

----------------------------------------------------------------------------
TypeError                                 Traceback (most recent call last)
<ipython-input-1-0b7a948d6b86> in <module>
     19 if __name__ == "__main__":
     20     arr = np.random.random((3,))
---> 21     reproducer(arr)
     22

<path>/dispatcher.py in _compile_for_args(self, *args, **kws)
    418                     e.patch_message('\n'.join((str(e).rstrip(), help_msg)))
    419             # ignore the FULL_TRACEBACKS config, this needs reporting!
--> 420             raise e
    421
    422     def inspect_llvm(self, signature=None):

<path>/dispatcher.py in _compile_for_args(self, *args, **kws)
    351                 argtypes.append(self.typeof_pyval(a))
    352         try:
--> 353             return self.compile(tuple(argtypes))
    354         except errors.ForceLiteralArg as e:
    355             # Received request for compiler re-entry with the list of arguments

<path>/compiler_lock.py in _acquire_compile_lock(*args, **kwargs)
     30         def _acquire_compile_lock(*args, **kwargs):
     31             with self:
---> 32                 return func(*args, **kwargs)
     33         return _acquire_compile_lock
     34

<path>/dispatcher.py in compile(self, sig)
    766             self._cache_misses[sig] += 1
    767             try:
--> 768                 cres = self._compiler.compile(args, return_type)
    769             except errors.ForceLiteralArg as e:
    770                 def folded(args, kws):

<path>/dispatcher.py in compile(self, args, return_type)
     75
     76     def compile(self, args, return_type):
---> 77         status, retval = self._compile_cached(args, return_type)
     78         if status:
     79             return retval

<path>/dispatcher.py in _compile_cached(self, args, return_type)
     89
     90         try:
---> 91             retval = self._compile_core(args, return_type)
     92         except errors.TypingError as e:
     93             self._failed_cache[key] = e

<path>/dispatcher.py in _compile_core(self, args, return_type)
    107                                       args=args, return_type=return_type,
    108                                       flags=flags, locals=self.locals,
--> 109                                       pipeline_class=self.pipeline_class)
    110         # Check typing error if object mode is used
    111         if cres.typing_error is not None and not flags.enable_pyobject:

<path>/compiler.py in compile_extra(typingctx, targetctx, func, args, return_type, flags, locals, library, pipeline_class)
    549     pipeline = pipeline_class(typingctx, targetctx, library,
    550                               args, return_type, flags, locals)
--> 551     return pipeline.compile_extra(func)
    552
    553

<path>/compiler.py in compile_extra(self, func)
    329         self.state.lifted = ()
    330         self.state.lifted_from = None
--> 331         return self._compile_bytecode()
    332
    333     def compile_ir(self, func_ir, lifted=(), lifted_from=None):

<path>/compiler.py in _compile_bytecode(self)
    391         """
    392         assert self.state.func_ir is None
--> 393         return self._compile_core()
    394
    395     def _compile_ir(self):

<path>/compiler.py in _compile_core(self)
    371                 self.state.status.fail_reason = e
    372                 if is_final_pipeline:
--> 373                     raise e
    374         else:
    375             raise CompilerError("All available pipelines exhausted")

<path>/compiler.py in _compile_core(self)
    362             res = None
    363             try:
--> 364                 pm.run(self.state)
    365                 if self.state.cr is not None:
    366                     break

<path>/compiler_machinery.py in run(self, state)
    345                     (self.pipeline_name, pass_desc)
    346                 patched_exception = self._patch_error(msg, e)
--> 347                 raise patched_exception
    348
    349     def dependency_analysis(self):

<path>/compiler_machinery.py in run(self, state)
    336                 pass_inst = _pass_registry.get(pss).pass_inst
    337                 if isinstance(pass_inst, CompilerPass):
--> 338                     self._runPass(idx, pass_inst, state)
    339                 else:
    340                     raise BaseException("Legacy pass in use")

<path>/compiler_lock.py in _acquire_compile_lock(*args, **kwargs)
     30         def _acquire_compile_lock(*args, **kwargs):
     31             with self:
---> 32                 return func(*args, **kwargs)
     33         return _acquire_compile_lock
     34

<path>/compiler_machinery.py in _runPass(self, index, pss, internal_state)
    300             mutated |= check(pss.run_initialization, internal_state)
    301         with SimpleTimer() as pass_time:
--> 302             mutated |= check(pss.run_pass, internal_state)
    303         with SimpleTimer() as finalize_time:
    304             mutated |= check(pss.run_finalizer, internal_state)

<path>/compiler_machinery.py in check(func, compiler_state)
    273
    274         def check(func, compiler_state):
--> 275             mangled = func(compiler_state)
    276             if mangled not in (True, False):
    277                 msg = ("CompilerPass implementations should return True/False. "

<path>/typed_passes.py in run_pass(self, state)
    267                                          state.flags,
    268                                          state.parfor_diagnostics)
--> 269         parfor_pass.run()
    270
    271         # check the parfor pass worked and warn if it didn't

<path>/parfor.py in run(self)
   1498         # run array analysis, a pre-requisite for parfor translation
   1499         remove_dels(self.func_ir.blocks)
-> 1500         self.array_analysis.run(self.func_ir.blocks)
   1501         # run stencil translation to parfor
   1502         if self.options.stencil:

<path>/array_analysis.py in run(self, blocks, equiv_set)
   1076                     inst = ir.SetItem(inst.target, inst.index_var, inst.value, inst.loc)
   1077                     self.calltypes[inst] = orig_calltype
-> 1078                 pre, post = self._analyze_inst(label, scope, equiv_set, inst, redefined)
   1079                 # Remove anything multiply defined in this block from every block equivs.
   1080                 if len(redefined) > 0:

<path>/array_analysis.py in _analyze_inst(self, label, scope, equiv_set, inst, redefined)
   1119                 shape = ()
   1120             elif isinstance(inst.value, ir.Expr):
-> 1121                 result = self._analyze_expr(scope, equiv_set, inst.value)
   1122                 if result:
   1123                     shape = result[0]

<path>/array_analysis.py in _analyze_expr(self, scope, equiv_set, expr)
   1294         except AttributeError:
   1295             return None
-> 1296         return guard(fn, scope, equiv_set, expr)
   1297
   1298     def _analyze_op_getattr(self, scope, equiv_set, expr):

<path>/ir_utils.py in guard(func, *args, **kwargs)
   1435     """
   1436     try:
-> 1437         return func(*args, **kwargs)
   1438     except GuardException:
   1439         return None

<path>/array_analysis.py in _analyze_op_getitem(self, scope, equiv_set, expr)
   1672
   1673     def _analyze_op_getitem(self, scope, equiv_set, expr):
-> 1674         result = self._index_to_shape(scope, equiv_set, expr.value, expr.index)
   1675         if result[0] is not None:
   1676             expr.index = result[0]

<path>/array_analysis.py in _index_to_shape(self, scope, equiv_set, var, ind_var)
   1632         for (typ, size, dsize) in zip(seq_typs, ind_shape, var_shape):
   1633             # Convert the given dimension of the get/setitem index expr.
-> 1634             shape_part, index_var_part = to_shape(typ, size, dsize)
   1635             shape_list.append(shape_part)
   1636

<path>/array_analysis.py in to_shape(typ, index, dsize)
   1619         def to_shape(typ, index, dsize):
   1620             if isinstance(typ, types.SliceType):
-> 1621                 return self.slice_size(index, dsize, equiv_set, scope, stmts)
   1622             elif isinstance(typ, types.Number):
   1623                 return None, None

<path>/array_analysis.py in slice_size(self, index, dsize, equiv_set, scope, stmts)
   1557         post_wrap_size_val = ir.Expr.binop(operator.sub, var2, var1, loc=loc)
   1558         self.calltypes[post_wrap_size_val] = signature(slice_typ, var2_typ, var1_typ)
-> 1559         self._define(equiv_set, post_wrap_size_var, slice_typ, post_wrap_size_val)
   1560
   1561         stmts.append(ir.Assign(value=size_val, target=size_var, loc=loc))

<path>/array_analysis.py in _define(self, equiv_set, var, typ, value)
   1105         self.func_ir._definitions[var.name] = [value]
   1106         redefineds = set()
-> 1107         equiv_set.define(var, redefineds, self.func_ir, typ)
   1108
   1109     def _analyze_inst(self, label, scope, equiv_set, inst, redefined):

<path>/array_analysis.py in define(self, var, redefined, func_ir, typ)
    838         if (func_ir and self.defs.get(name, 0) == 1 and
    839                 isinstance(typ, types.Number)):
--> 840             value = guard(self._get_or_set_rel, name, func_ir)
    841             # turn constant definition into equivalence
    842             if isinstance(value, int):

<path>/ir_utils.py in guard(func, *args, **kwargs)
   1435     """
   1436     try:
-> 1437         return func(*args, **kwargs)
   1438     except GuardException:
   1439         return None

<path>/array_analysis.py in _get_or_set_rel(self, name, func_ir)
    796                         value = plus(lhs, rhs)
    797                     elif expr.fn == operator.sub:
--> 798                         value = minus(lhs, rhs)
    799             elif isinstance(expr, ir.Const) and isinstance(expr.value, int):
    800                 value = expr.value

<path>/array_analysis.py in minus(x, y)
    768             def minus(x, y):
    769                 if isinstance(y, int):
--> 770                     return plus(x, -y)
    771                 elif (isinstance(x, tuple) and isinstance(y, tuple) and
    772                       x[0] == y[0]):

<path>/array_analysis.py in plus(x, y)
    761                         return (var, x + offset)
    762                 else:
--> 763                     (var, offset) = x
    764                     if y_is_const:
    765                         return (var, y + offset)

TypeError: Failed in nopython mode pipeline (step: convert to parfors)
cannot unpack non-iterable NoneType object
```
</details>
This looks familiar, I think there's another one of these cases reported somewhere.
xref https://github.com/numba/numba/issues/5439

Resolving patch:
diff --git a/numba/parfors/array_analysis.py b/numba/parfors/array_analysis.py
--- a/numba/parfors/array_analysis.py
+++ b/numba/parfors/array_analysis.py
@@ -855,6 +855,11 @@ def minus(x, y):
                         index = tuple(
                             self.obj_to_ind.get(x.name, -1) for x in expr.args
                         )
+                        # If wrap_index for a slice works on a variable
+                        # that is not analyzable (e.g., multiple definitions)
+                        # then we have to return None here since we can't know
+                        # how that size will compare to others if we can't
+                        # analyze some part of the slice.
                         if -1 in index:
                             return None
                         names = self.ext_shapes.get(index, [])
@@ -865,7 +870,11 @@ def minus(x, y):
                 elif expr.op == "binop":
                     lhs = self._get_or_set_rel(expr.lhs, func_ir)
                     rhs = self._get_or_set_rel(expr.rhs, func_ir)
-                    if expr.fn == operator.add:
+                    # If either the lhs or rhs is not analyzable
+                    # then don't try to record information this var.
+                    if lhs is None or rhs is None:
+                        return None
+                    elif expr.fn == operator.add:
                         value = plus(lhs, rhs)
                     elif expr.fn == operator.sub:
                         value = minus(lhs, rhs)
@@ -931,6 +940,7 @@ def define(self, var, redefined, func_ir=None, typ=None):
                     self.ind_to_var[ind].append(var)
                 else:
                     self.ind_to_var[ind] = [var]
+            return True
 
     def _insert(self, objs):
         """Overload _insert method to handle ind changes between relative
@@ -1225,7 +1235,8 @@ def _define(self, equiv_set, var, typ, value):
         self.typemap[var.name] = typ
         self.func_ir._definitions[var.name] = [value]
         redefineds = set()
-        equiv_set.define(var, redefineds, self.func_ir, typ)
+        # Propagate the success or failure of define.
+        return equiv_set.define(var, redefineds, self.func_ir, typ)
 
     def _analyze_inst(self, label, scope, equiv_set, inst, redefined):
         pre = []
@@ -1833,6 +1844,8 @@ def gen_wrap_if_not_known(val, val_typ, known):
                 var = ir.Var(scope, mk_unique_var("var"), loc)
                 var_typ = types.intp
                 new_value = ir.Expr.call(wrap_var, [val, dsize], {}, loc)
+                # def_res will be False if there is something unanalyzable
+                # that prevents a size association from being created.
                 self._define(equiv_set, var, var_typ, new_value)
                 self.calltypes[new_value] = sig
                 return (var, var_typ, new_value)
@@ -1842,10 +1855,20 @@ def gen_wrap_if_not_known(val, val_typ, known):
         var1, var1_typ, value1 = gen_wrap_if_not_known(lhs, lhs_typ, lhs_known)
         var2, var2_typ, value2 = gen_wrap_if_not_known(rhs, rhs_typ, rhs_known)
 
+        stmts.append(ir.Assign(value=size_val, target=size_var, loc=loc))
+        stmts.append(ir.Assign(value=wrap_def, target=wrap_var, loc=loc))
+        if value1 is not None:
+            stmts.append(ir.Assign(value=value1, target=var1, loc=loc))
+        if value2 is not None:
+            stmts.append(ir.Assign(value=value2, target=var2, loc=loc))
+
         post_wrap_size_var = ir.Var(
             scope, mk_unique_var("post_wrap_slice_size"), loc
         )
-        post_wrap_size_val = ir.Expr.binop(operator.sub, var2, var1, loc=loc)
+        post_wrap_size_val = ir.Expr.binop(operator.sub,
+                                           var2,
+                                           var1,
+                                           loc=loc)
         self.calltypes[post_wrap_size_val] = signature(
             slice_typ, var2_typ, var1_typ
         )
@@ -1853,12 +1876,6 @@ def gen_wrap_if_not_known(val, val_typ, known):
             equiv_set, post_wrap_size_var, slice_typ, post_wrap_size_val
         )
 
-        stmts.append(ir.Assign(value=size_val, target=size_var, loc=loc))
-        stmts.append(ir.Assign(value=wrap_def, target=wrap_var, loc=loc))
-        if value1 is not None:
-            stmts.append(ir.Assign(value=value1, target=var1, loc=loc))
-        if value2 is not None:
-            stmts.append(ir.Assign(value=value2, target=var2, loc=loc))
         stmts.append(
             ir.Assign(
                 value=post_wrap_size_val, target=post_wrap_size_var, loc=loc


Test patch:
diff --git a/numba/tests/test_parfors.py b/numba/tests/test_parfors.py
--- a/numba/tests/test_parfors.py
+++ b/numba/tests/test_parfors.py
@@ -2983,6 +2983,25 @@ def test_impl(a):
 
         self.check(test_impl, np.arange(4))
 
+    @skip_parfors_unsupported
+    def test_parfor_slice27(self):
+        # issue5601: tests array analysis of the slice with
+        # n_valid_vals of unknown size.
+        def test_impl(a):
+            n_valid_vals = 0
+
+            for i in prange(a.shape[0]):
+                if a[i] != 0:
+                    n_valid_vals += 1
+
+                if n_valid_vals:
+                    unused = a[:n_valid_vals]
+
+            return 0
+
+        self.check(test_impl, np.arange(3))
+
+
 class TestParforsOptions(TestParforsBase):
 
     def check(self, pyfunc, *args, **kwargs):