Repository: numba/numba

Issue:
0.52.0rc1 bug in array analysis for tuples of multi-dimensional arrays
<!--

Thanks for opening an issue! To help the Numba team handle your information
efficiently, please first ensure that there is no other issue present that
already describes the issue you have
(search at https://github.com/numba/numba/issues?&q=is%3Aissue).

-->

## Reporting a bug

<!--

Before submitting a bug report please ensure that you can check off these boxes:

-->

- [x] I have tried using the latest released version of Numba (most recent is
 visible in the change log (https://github.com/numba/numba/blob/master/CHANGE_LOG).
- [x] I have included below a minimal working reproducer (if you are unsure how
 to write one see http://matthewrocklin.com/blog/work/2018/02/28/minimal-bug-reports).

<!--

Please include details of the bug here, including, if applicable, what you
expected to happen!

-->
Looks like tuples of multi-dimensional arrays don't keep their shapes properly in 0.52.0rc1:

```python
In [1]: @numba.njit(parallel=True)
   ...: def f(A):
   ...:     S = (A, A)
   ...:     return S[0].sum()
   ...:

In [2]: f(np.ones((3, 3)))
---------------------------------------------------------------------------
IndexError                                Traceback (most recent call last)
<ipython-input-2-c76d1478a056> in <module>
----> 1 f(np.ones((3, 3)))

~/dev/numba/numba/core/dispatcher.py in _compile_for_args(self, *args, **kws)
    431                     e.patch_message('\n'.join((str(e).rstrip(), help_msg)))
    432             # ignore the FULL_TRACEBACKS config, this needs reporting!
--> 433             raise e
    434
    435     def inspect_llvm(self, signature=None):

~/dev/numba/numba/core/dispatcher.py in _compile_for_args(self, *args, **kws)
    364                 argtypes.append(self.typeof_pyval(a))
    365         try:
--> 366             return self.compile(tuple(argtypes))
    367         except errors.ForceLiteralArg as e:
    368             # Received request for compiler re-entry with the list of arguments

~/dev/numba/numba/core/compiler_lock.py in _acquire_compile_lock(*args, **kwargs)
     30         def _acquire_compile_lock(*args, **kwargs):
     31             with self:
---> 32                 return func(*args, **kwargs)
     33         return _acquire_compile_lock
     34

~/dev/numba/numba/core/dispatcher.py in compile(self, sig)
    855             self._cache_misses[sig] += 1
    856             try:
--> 857                 cres = self._compiler.compile(args, return_type)
    858             except errors.ForceLiteralArg as e:
    859                 def folded(args, kws):

~/dev/numba/numba/core/dispatcher.py in compile(self, args, return_type)
     75
     76     def compile(self, args, return_type):
---> 77         status, retval = self._compile_cached(args, return_type)
     78         if status:
     79             return retval

~/dev/numba/numba/core/dispatcher.py in _compile_cached(self, args, return_type)
     89
     90         try:
---> 91             retval = self._compile_core(args, return_type)
     92         except errors.TypingError as e:
     93             self._failed_cache[key] = e

~/dev/numba/numba/core/dispatcher.py in _compile_core(self, args, return_type)
    102
    103         impl = self._get_implementation(args, {})
--> 104         cres = compiler.compile_extra(self.targetdescr.typing_context,
    105                                       self.targetdescr.target_context,
    106                                       impl,

~/dev/numba/numba/core/compiler.py in compile_extra(typingctx, targetctx, func, args, return_type, flags, locals, library, pipeline_class)
    600     pipeline = pipeline_class(typingctx, targetctx, library,
    601                               args, return_type, flags, locals)
--> 602     return pipeline.compile_extra(func)
    603
    604

~/dev/numba/numba/core/compiler.py in compile_extra(self, func)
    350         self.state.lifted = ()
    351         self.state.lifted_from = None
--> 352         return self._compile_bytecode()
    353
    354     def compile_ir(self, func_ir, lifted=(), lifted_from=None):

~/dev/numba/numba/core/compiler.py in _compile_bytecode(self)
    412         """
    413         assert self.state.func_ir is None
--> 414         return self._compile_core()
    415
    416     def _compile_ir(self):

~/dev/numba/numba/core/compiler.py in _compile_core(self)
    392                 self.state.status.fail_reason = e
    393                 if is_final_pipeline:
--> 394                     raise e
    395         else:
    396             raise CompilerError("All available pipelines exhausted")

~/dev/numba/numba/core/compiler.py in _compile_core(self)
    383             res = None
    384             try:
--> 385                 pm.run(self.state)
    386                 if self.state.cr is not None:
    387                     break

~/dev/numba/numba/core/compiler_machinery.py in run(self, state)
    337                     (self.pipeline_name, pass_desc)
    338                 patched_exception = self._patch_error(msg, e)
--> 339                 raise patched_exception
    340
    341     def dependency_analysis(self):

~/dev/numba/numba/core/compiler_machinery.py in run(self, state)
    328                 pass_inst = _pass_registry.get(pss).pass_inst
    329                 if isinstance(pass_inst, CompilerPass):
--> 330                     self._runPass(idx, pass_inst, state)
    331                 else:
    332                     raise BaseException("Legacy pass in use")

~/dev/numba/numba/core/compiler_lock.py in _acquire_compile_lock(*args, **kwargs)
     30         def _acquire_compile_lock(*args, **kwargs):
     31             with self:
---> 32                 return func(*args, **kwargs)
     33         return _acquire_compile_lock
     34

~/dev/numba/numba/core/compiler_machinery.py in _runPass(self, index, pss, internal_state)
    287             mutated |= check(pss.run_initialization, internal_state)
    288         with SimpleTimer() as pass_time:
--> 289             mutated |= check(pss.run_pass, internal_state)
    290         with SimpleTimer() as finalize_time:
    291             mutated |= check(pss.run_finalizer, internal_state)

~/dev/numba/numba/core/compiler_machinery.py in check(func, compiler_state)
    260
    261         def check(func, compiler_state):
--> 262             mangled = func(compiler_state)
    263             if mangled not in (True, False):
    264                 msg = ("CompilerPass implementations should return True/False. "

~/dev/numba/numba/core/typed_passes.py in run_pass(self, state)
    288                                          state.flags,
    289                                          state.parfor_diagnostics)
--> 290         parfor_pass.run()
    291
    292         remove_dels(state.func_ir.blocks)

~/dev/numba/numba/parfors/parfor.py in run(self)
   2715         """run parfor conversion pass: replace Numpy calls
   2716         with Parfors when possible and optimize the IR."""
-> 2717         self._pre_run()
   2718         # run stencil translation to parfor
   2719         if self.options.stencil:

~/dev/numba/numba/parfors/parfor.py in _pre_run(self)
   2707     def _pre_run(self):
   2708         # run array analysis, a pre-requisite for parfor translation
-> 2709         self.array_analysis.run(self.func_ir.blocks)
   2710         # NOTE: Prepare _max_label. See #6102
   2711         ir_utils._max_label = max(ir_utils._max_label,

~/dev/numba/numba/parfors/array_analysis.py in run(self, blocks, equiv_set)
   1118         topo_order = find_topo_order(blocks, cfg=cfg)
   1119         # Traverse blocks in topological order
-> 1120         self._run_on_blocks(topo_order, blocks, cfg, init_equiv_set)
   1121
   1122         if config.DEBUG_ARRAY_OPT >= 1:

~/dev/numba/numba/parfors/array_analysis.py in _run_on_blocks(self, topo_order, blocks, cfg, init_equiv_set)
   1138             block = blocks[label]
   1139             scope = block.scope
-> 1140             pending_transforms = self._determine_transform(
   1141                 cfg, block, label, scope, init_equiv_set
   1142             )

~/dev/numba/numba/parfors/array_analysis.py in _determine_transform(self, cfg, block, label, scope, init_equiv_set)
   1219                 )
   1220                 self.calltypes[inst] = orig_calltype
-> 1221             pre, post = self._analyze_inst(
   1222                 label, scope, equiv_set, inst, redefined
   1223             )

~/dev/numba/numba/parfors/array_analysis.py in _analyze_inst(self, label, scope, equiv_set, inst, redefined)
   1311                     )
   1312                 ):
-> 1313                     (shape, post) = self._gen_shape_call(
   1314                         equiv_set, lhs, typ.ndim, shape
   1315                     )

~/dev/numba/numba/parfors/array_analysis.py in _gen_shape_call(self, equiv_set, var, ndims, shape)
   2973         for i in range(ndims):
   2974             skip = False
-> 2975             if shape and shape[i]:
   2976                 if isinstance(shape[i], ir.Var):
   2977                     typ = self.typemap[shape[i].name]

IndexError: Failed in nopython mode pipeline (step: convert to parfors)
tuple index out of range
```


Discussion:
I can reproduce the issue. Copy-pastable reproducer:

```python
from numba import njit
import numpy as np

@njit(parallel=True)
def f(A):
    S = (A, A)
    return S[0].sum()


f(np.ones((3, 3)))
```

This seems to only apply to the ParallelAccelerator - removing `parallel=True` makes the issue disappear.
I've placed this in the 0.52.0RC2 milestone as a reminder that we might want to check whether this is a regression that we want to fix for the next RC.
Has this ever worked? I've just checked 0.48 - 0.52.0rc1 and all fail the same way?
> Has this ever worked? I've just checked 0.48 - 0.52.0rc1 and all fail the same way?

OK - I've removed it from the milestone.
This showed up in a test in our CI that worked previously. Maybe previous versions work for this reproducer by coincidence (could have other possible reproducers).
> This showed up in a test in our CI that worked previously. Maybe previous versions work for this reproducer by coincidence (could have other possible reproducers).

Knowing the patches that went into 0.52.0rc1, it's conceivable that there is a bug in relation to this, but without a reproducer for a regression I'm not sure there's anything to fix?

@ehsantn if you can find a reproducer from your CI tests that breaks on the RC but worked in releases prior to the RC then someone can take a look. Thanks for your help.
Ok, I have a reproducer that works on previous versions:
```python
@numba.njit(parallel=True)
def f(A):
    S = (A,) + (A, A)
    return S[0]

f(np.ones((3,3)))
```
Looks like the recent support for tuple addition exposes the problem (which already existed).
> Ok, I have a reproducer that works on previous versions:
> 
> ```python
> @numba.njit(parallel=True)
> def f(A):
>     S = (A,) + (A, A)
>     return S[0]
> 
> f(np.ones((3,3)))
> ```
> 
> Looks like the recent support for tuple addition exposes the problem (which already existed).

Thanks for the reproducer. This function has no parallel transform possible, but triggers the problem in array analysis. CC @DrTodd13 please could you take a look? thanks.

Resolving patch:
diff --git a/numba/parfors/array_analysis.py b/numba/parfors/array_analysis.py
--- a/numba/parfors/array_analysis.py
+++ b/numba/parfors/array_analysis.py
@@ -2064,6 +2064,17 @@ def _analyze_op_arrayexpr(self, scope, equiv_set, expr):
         )
 
     def _analyze_op_build_tuple(self, scope, equiv_set, expr):
+        # For the moment, we can't do anything with tuples that
+        # contain arrays, compared to array dimensions.  Return
+        # None to say we won't track this tuple if a part of it
+        # is an array.
+        for x in expr.items:
+            if (
+                isinstance(x, ir.Var)
+                and isinstance(self.typemap[x.name], types.ArrayCompatible)
+            ):
+                return None
+
         consts = []
         for var in expr.items:
             x = guard(find_const, self.func_ir, var)


Test patch:
diff --git a/numba/tests/test_parfors.py b/numba/tests/test_parfors.py
--- a/numba/tests/test_parfors.py
+++ b/numba/tests/test_parfors.py
@@ -1722,6 +1722,16 @@ def test_impl(a):
         x = np.array([1, 1])
         self.check(test_impl, x)
 
+    @skip_parfors_unsupported
+    def test_array_tuple_concat(self):
+        # issue6399
+        def test_impl(a):
+            S = (a,) + (a, a)
+            return S[0].sum()
+
+        x = np.ones((3,3))
+        self.check(test_impl, x)
+
 
 class TestParforsLeaks(MemoryLeakMixin, TestParforsBase):
     def check(self, pyfunc, *args, **kwargs):