Repository: numba/numba

Issue:
suspected parfors array analysis failure under `reshape`
- [x] I am using the latest released version of Numba (most recent is visible in
 the change log (https://github.com/numba/numba/blob/master/CHANGE_LOG).
- [x] I have included below a minimal working reproducer (if you are unsure how
 to write one see http://matthewrocklin.com/blog/work/2018/02/28/minimal-bug-reports).

The below fails with: `AssertionError: Sizes of result_matrix, result_data do not match on...` under the `parallel=True` function. I suspect that this is a Parfors array analysis failure. Note the use of `-1` in reshape is permitted in a single dimension to indicate "infer from other specified dimensions", I think not handling this is the root cause of the problem. Original report on gitter [here](https://gitter.im/numba/numba?at=5b9b9badf08bc22dfb56c8ec):

```python
import numpy as np
from numba import njit

n_dates = 4
n_sec = 3
price_vector = np.array([1.0, 2.0, 3.0, 4.0, 5.0, 
                        6.0, 7.0, 8.0, 9.0, 10.0,
                        11.0, 12.0])

def func(vec, n_sec, n_dates):
    result_matrix = np.zeros((n_sec, n_sec, n_dates), dtype=np.float64)
    for i in range(n_dates):
        sub_vec = vec[(i * n_sec):((i+1)*n_sec)]
        z = sub_vec.reshape(-1, 1)
        print(z.shape)
        result_data = sub_vec / z
        result_matrix[:,:,i] = result_data

    return result_matrix

print(njit(func)(price_vector, n_sec, n_dates))
print(njit(parallel=True)(func)(price_vector, n_sec, n_dates))
```


Discussion:
Is there an update on this? I posted the original question on gitter a while ago. This bug prevents me from running my code on multi-cores and is therefore slower than it could be. Although the code works, the added performance would be great! 
@qlandman I'm afraid not. @DrTodd13 any ideas? If this is just array equivalence analysis failing I can probably take a look.
@qlandman @stuartarchibald The problem is with array analysis.  The reshape call allows "-1" to mean infer this length but our current array analysis doesn't handle this special meaning of -1 and treats it naturally so it thinks the array length is -1.  Then there's a later assert that two array lengths are equal, which they won't be because that one is -1.  Working on a solution.

Resolving patch:
diff --git a/numba/array_analysis.py b/numba/array_analysis.py
--- a/numba/array_analysis.py
+++ b/numba/array_analysis.py
@@ -1268,12 +1268,15 @@ def _analyze_op_call(self, scope, equiv_set, expr):
 
         fname, mod_name = find_callname(
             self.func_ir, expr, typemap=self.typemap)
+        added_mod_name = False
         # call via attribute (i.e. array.func)
         if (isinstance(mod_name, ir.Var)
                 and isinstance(self.typemap[mod_name.name],
                                 types.ArrayCompatible)):
             args = [mod_name] + expr.args
             mod_name = 'numpy'
+            # Remember that args and expr.args don't alias.
+            added_mod_name = True
         else:
             args = expr.args
         fname = "_analyze_op_call_{}_{}".format(
@@ -1285,7 +1288,14 @@ def _analyze_op_call(self, scope, equiv_set, expr):
                 fn = getattr(self, fname)
             except AttributeError:
                 return None
-            return guard(fn, scope, equiv_set, args, dict(expr.kws))
+            result = guard(fn, scope, equiv_set, args, dict(expr.kws))
+            # We want the ability for function fn to modify arguments.
+            # If args and expr.args don't alias then we need the extra
+            # step of assigning back into expr.args from the args that
+            # was passed to fn.
+            if added_mod_name:
+                expr.args = args[1:]
+            return result
 
     def _analyze_op_call___builtin___len(self, scope, equiv_set, args, kws):
         # python 2 version of len()
@@ -1422,7 +1432,60 @@ def _analyze_op_call_numpy_reshape(self, scope, equiv_set, args, kws):
             typ = self.typemap[args[1].name]
             if isinstance(typ, types.BaseTuple):
                 return args[1], []
-        return tuple(args[1:]), []
+
+        # Reshape is allowed to take one argument that has the value <0.
+        # This means that the size of that dimension should be inferred from
+        # the size of the array being reshaped and the other dimensions
+        # specified.  Our general approach here is to see if the reshape
+        # has any <0 arguments.  If it has more than one then throw a
+        # ValueError.  If exactly one <0 argument is found, remember its
+        # argument index.
+        stmts = []
+        neg_one_index = -1
+        for arg_index in range(1, len(args)):
+            reshape_arg = args[arg_index]
+            reshape_arg_def = guard(get_definition, self.func_ir, reshape_arg)
+            if isinstance(reshape_arg_def, ir.Const):
+                if reshape_arg_def.value < 0:
+                    if neg_one_index == -1:
+                        neg_one_index = arg_index
+                    else:
+                        msg = ("The reshape API may only include one negative"
+                               " argument. %s" % str(reshape_arg.loc))
+                        raise ValueError(msg)
+
+        if neg_one_index >= 0:
+            # If exactly one <0 argument to reshape was found, then we are
+            # going to insert code to calculate the missing dimension and then
+            # replace the negative with the calculated size.  We do this because we
+            # can't let array equivalence analysis think that some array has
+            # a negative dimension size.
+            loc = args[0].loc
+            # Create a variable to hold the size of the array being reshaped.
+            calc_size_var = ir.Var(scope, mk_unique_var("calc_size_var"), loc)
+            self.typemap[calc_size_var.name] = types.intp
+            # Assign the size of the array calc_size_var.
+            init_calc_var = ir.Assign(ir.Expr.getattr(args[0], "size", loc), calc_size_var, loc)
+            stmts.append(init_calc_var)
+            # For each other dimension, divide the current size by the specified
+            # dimension size.  Once all such dimensions have been done then what is
+            # left is the size of the negative dimension.
+            for arg_index in range(1, len(args)):
+                # Skip the negative dimension.
+                if arg_index == neg_one_index:
+                    continue
+                div_calc_size_var = ir.Var(scope, mk_unique_var("calc_size_var"), loc)
+                self.typemap[div_calc_size_var.name] = types.intp
+                # Calculate the next size as current size // the current arg's dimension size.
+                new_binop = ir.Expr.binop(operator.floordiv, calc_size_var, args[arg_index], loc)
+                div_calc = ir.Assign(new_binop, div_calc_size_var, loc)
+                self.calltypes[new_binop] = signature(types.intp, types.intp, types.intp)
+                stmts.append(div_calc)
+                calc_size_var = div_calc_size_var
+            # Put the calculated value back into the reshape arguments, replacing the negative.
+            args[neg_one_index] = div_calc_size_var
+
+        return tuple(args[1:]), stmts
 
     def _analyze_op_call_numpy_transpose(self, scope, equiv_set, args, kws):
         in_arr = args[0]


Test patch:
diff --git a/numba/tests/test_parfors.py b/numba/tests/test_parfors.py
--- a/numba/tests/test_parfors.py
+++ b/numba/tests/test_parfors.py
@@ -1403,6 +1403,67 @@ def test_impl():
 
         self.check(test_impl)
 
+    @skip_unsupported
+    def test_reshape_with_neg_one(self):
+        # issue3314
+        def test_impl(a, b):
+            result_matrix = np.zeros((b, b, 1), dtype=np.float64)
+            sub_a = a[0:b]
+            a = sub_a.size
+            b = a / 1
+            z = sub_a.reshape(-1, 1)
+            result_data = sub_a / z
+            result_matrix[:,:,0] = result_data
+            return result_matrix
+
+        a = np.array([1.0, 2.0, 3.0, 4.0, 5.0, 6.0,
+                   7.0, 8.0, 9.0, 10.0, 11.0, 12.0])
+        b = 3
+
+        self.check(test_impl, a, b)
+
+    @skip_unsupported
+    def test_reshape_with_large_neg(self):
+        # issue3314
+        def test_impl(a, b):
+            result_matrix = np.zeros((b, b, 1), dtype=np.float64)
+            sub_a = a[0:b]
+            a = sub_a.size
+            b = a / 1
+            z = sub_a.reshape(-1307, 1)
+            result_data = sub_a / z
+            result_matrix[:,:,0] = result_data
+            return result_matrix
+
+        a = np.array([1.0, 2.0, 3.0, 4.0, 5.0, 6.0,
+                   7.0, 8.0, 9.0, 10.0, 11.0, 12.0])
+        b = 3
+
+        self.check(test_impl, a, b)
+
+    @skip_unsupported
+    def test_reshape_with_too_many_neg_one(self):
+        # issue3314
+        with self.assertRaises(ValueError) as raised:
+            @njit(parallel=True)
+            def test_impl(a, b):
+                rm = np.zeros((b, b, 1), dtype=np.float64)
+                sub_a = a[0:b]
+                a = sub_a.size
+                b = a / 1
+                z = sub_a.reshape(-1, -1)
+                result_data = sub_a / z
+                rm[:,:,0] = result_data
+                return rm
+
+            a = np.array([1.0, 2.0, 3.0, 4.0, 5.0, 6.0,
+                       7.0, 8.0, 9.0, 10.0, 11.0, 12.0])
+            b = 3
+            test_impl(a, b)
+
+        msg = ("The reshape API may only include one negative argument.")
+        self.assertIn(msg, str(raised.exception))
+
 class TestPrangeBase(TestParforsBase):
 
     def __init__(self, *args):