Repository: numba/numba

Issue:
prange fails when array are changed with augmented assignment
**Code sample:**
```python
import numpy as np
import numba

NUM = 100

@numba.jit(nopython=True, parallel=True)
def numba_array_unary_array():
    result = np.zeros(1, np.int_)

    for i in numba.prange(NUM):
        result += np.array([i], np.int_)

    return result

print(numba_array_unary_array()) # raise numba.errors.LoweringError
```
While I was investigating `prange`'s behavior, I encountered this error. As far as I understand, it's not good to assign values to array without slice ~~or with integer indexer~~ in loops of prange (am I correct?). But the traceback let me report, so I did just in case.
(python==3.6.5, numba==0.38.0) **EDIT:** checked this can be reproduced in 0.39.0rc1
　　
**Traceback:**
<details>  

```python
Traceback (most recent call last):
  File "C:\Users\kittoku\Anaconda3\envs\test_env\lib\site-packages\numba\errors.py", line 491, in new_error_context
    yield
  File "C:\Users\kittoku\Anaconda3\envs\test_env\lib\site-packages\numba\lowering.py", line 216, in lower_block
    self.lower_inst(inst)
  File "C:\Users\kittoku\Anaconda3\envs\test_env\lib\site-packages\numba\lowering.py", line 365, in lower_inst
    func(self, inst)
  File "C:\Users\kittoku\Anaconda3\envs\test_env\lib\site-packages\numba\npyufunc\parfor.py", line 83, in _lower_parfor_parallel
    bool(alias_map), index_var_typ)
  File "C:\Users\kittoku\Anaconda3\envs\test_env\lib\site-packages\numba\npyufunc\parfor.py", line 357, in _create_gufunc_for_parfor_body
    typemap[arr] = types.npytypes.Array(typemap[var], 1, "C")
  File "C:\Users\kittoku\Anaconda3\envs\test_env\lib\site-packages\numba\types\abstract.py", line 60, in __call__
    inst = type.__call__(cls, *args, **kwargs)
  File "C:\Users\kittoku\Anaconda3\envs\test_env\lib\site-packages\numba\types\npytypes.py", line 288, in __init__
    super(Array, self).__init__(dtype, ndim, layout, name=name)
  File "C:\Users\kittoku\Anaconda3\envs\test_env\lib\site-packages\numba\types\common.py", line 51, in __init__
    raise TypeError("Buffer dtype cannot be buffer")
TypeError: Buffer dtype cannot be buffer

During handling of the above exception, another exception occurred:

Traceback (most recent call last):
  File "<stdin>", line 1, in <module>
  File "C:\Users\kittoku\Anaconda3\envs\test_env\lib\site-packages\numba\dispatcher.py", line 360, in _compile_for_args
    raise e
  File "C:\Users\kittoku\Anaconda3\envs\test_env\lib\site-packages\numba\dispatcher.py", line 311, in _compile_for_args
    return self.compile(tuple(argtypes))
  File "C:\Users\kittoku\Anaconda3\envs\test_env\lib\site-packages\numba\dispatcher.py", line 618, in compile
    cres = self._compiler.compile(args, return_type)
  File "C:\Users\kittoku\Anaconda3\envs\test_env\lib\site-packages\numba\dispatcher.py", line 83, in compile
    pipeline_class=self.pipeline_class)
  File "C:\Users\kittoku\Anaconda3\envs\test_env\lib\site-packages\numba\compiler.py", line 871, in compile_extra
    return pipeline.compile_extra(func)
  File "C:\Users\kittoku\Anaconda3\envs\test_env\lib\site-packages\numba\compiler.py", line 365, in compile_extra
    return self._compile_bytecode()
  File "C:\Users\kittoku\Anaconda3\envs\test_env\lib\site-packages\numba\compiler.py", line 802, in _compile_bytecode
    return self._compile_core()
  File "C:\Users\kittoku\Anaconda3\envs\test_env\lib\site-packages\numba\compiler.py", line 789, in _compile_core
    res = pm.run(self.status)
  File "C:\Users\kittoku\Anaconda3\envs\test_env\lib\site-packages\numba\compiler.py", line 251, in run
    raise patched_exception
  File "C:\Users\kittoku\Anaconda3\envs\test_env\lib\site-packages\numba\compiler.py", line 243, in run
    stage()
  File "C:\Users\kittoku\Anaconda3\envs\test_env\lib\site-packages\numba\compiler.py", line 676, in stage_nopython_backend
    self._backend(lowerfn, objectmode=False)
  File "C:\Users\kittoku\Anaconda3\envs\test_env\lib\site-packages\numba\compiler.py", line 626, in _backend
    lowered = lowerfn()
  File "C:\Users\kittoku\Anaconda3\envs\test_env\lib\site-packages\numba\compiler.py", line 613, in backend_nopython_mode
    self.flags)
  File "C:\Users\kittoku\Anaconda3\envs\test_env\lib\site-packages\numba\compiler.py", line 990, in native_lowering_stage
    lower.lower()
  File "C:\Users\kittoku\Anaconda3\envs\test_env\lib\site-packages\numba\lowering.py", line 135, in lower
    self.lower_normal_function(self.fndesc)
  File "C:\Users\kittoku\Anaconda3\envs\test_env\lib\site-packages\numba\lowering.py", line 176, in lower_normal_function
    entry_block_tail = self.lower_function_body()
  File "C:\Users\kittoku\Anaconda3\envs\test_env\lib\site-packages\numba\lowering.py", line 201, in lower_function_body
    self.lower_block(block)
  File "C:\Users\kittoku\Anaconda3\envs\test_env\lib\site-packages\numba\lowering.py", line 216, in lower_block
    self.lower_inst(inst)
  File "C:\Users\kittoku\Anaconda3\envs\test_env\lib\contextlib.py", line 99, in __exit__
    self.gen.throw(type, value, traceback)
  File "C:\Users\kittoku\Anaconda3\envs\test_env\lib\site-packages\numba\errors.py", line 499, in new_error_context
    six.reraise(type(newerr), newerr, tb)
  File "C:\Users\kittoku\Anaconda3\envs\test_env\lib\site-packages\numba\six.py", line 659, in reraise
    raise value
numba.errors.LoweringError: Failed at nopython (nopython mode backend)
Buffer dtype cannot be buffer

File "<stdin>", line 5:

[1] During: lowering "id=6[LoopNest(index_variable = parfor_index.271, range = (0, $16.3, 1))]{28: <ir.Block at <stdin> (5)>}Var(parfor_index.271, <stdin> (5))" at <stdin> (5)
-------------------------------------------------------------------------------
This should not have happened, a problem has occurred in Numba's internals.

Please report the error message and traceback, along with a minimal reproducer
at: https://github.com/numba/numba/issues/new

If you need help writing a minimal reproducer please see:
http://matthewrocklin.com/blog/work/2018/02/28/minimal-bug-reports

If more help is needed please feel free to speak to the Numba core developers
directly at: https://gitter.im/numba/numba

Thanks in advance for your help in improving Numba!
```
</details>  

**All script:**
<details>
  
```python
import numpy as np
import numba

NUM = 100

@numba.jit(nopython=True, parallel=True)
def numba_scalar():
    result = 0

    for i in range(NUM):
        result += i

    return result

@numba.jit(nopython=True, parallel=True)
def numba_array_unary_scalar():
    result = np.zeros(1, np.int_)

    for i in numba.prange(NUM):
        result[0] += i

    return result

@numba.jit(nopython=True, parallel=True)
def numba_array_binary_array():
    result = np.zeros(1, np.int_)

    for i in numba.prange(NUM):
        result = result + np.array([i], np.int_)

    return result

@numba.jit(nopython=True, parallel=True)
def numba_array_unary_array():
    result = np.zeros(1, np.int_)

    for i in numba.prange(NUM):
        result += np.array([i], np.int_)

    return result


print(numba_scalar()) # -> 4950
print(numba_array_unary_scalar()) # -> [428]
print(numba_array_binary_array()) # -> [0]
print(numba_array_unary_array()) # raise numba.errors.LoweringError
```
</details>


Discussion:
Thanks for the report. I can reproduce. Also, this is broken:
```python
import numpy as np
import numba

NUM = 100

@numba.jit(nopython=True, parallel=True)
def numba_array_unary_array():
    result = np.zeros(1, np.int_)

    for i in numba.prange(NUM):
        result = result + np.array([i], np.int_)

    return result

print(numba_array_unary_array())
```
produces `[0]`.

@ehsantn @DrTodd13 any ideas? Thanks.
BTW, `numba_array_unary_scalar()` failed just because of race condition, not indexer.

Resolving patch:
diff --git a/numba/inline_closurecall.py b/numba/inline_closurecall.py
--- a/numba/inline_closurecall.py
+++ b/numba/inline_closurecall.py
@@ -939,37 +939,69 @@ def inline_array(array_var, expr, stmts, list_vars, dels):
                            ret_type.ndim == 1)
         loc = expr.loc
         list_var = expr.args[0]
+        # Get the type of the array to be created.
         array_typ = typemap[array_var.name]
         debug_print("inline array_var = ", array_var, " list_var = ", list_var)
+        # Get the element type of the array to be created.
         dtype = array_typ.dtype
-        seq, op = find_build_sequence(func_ir, list_var)
+        # Get the sequence of operations to provide values to the new array.
+        seq, _ = find_build_sequence(func_ir, list_var)
         size = len(seq)
+        # Create a tuple to pass to empty below to specify the new array size.
         size_var = ir.Var(scope, mk_unique_var("size"), loc)
         size_tuple_var = ir.Var(scope, mk_unique_var("size_tuple"), loc)
         size_typ = types.intp
         size_tuple_typ = types.UniTuple(size_typ, 1)
-
         typemap[size_var.name] = size_typ
         typemap[size_tuple_var.name] = size_tuple_typ
-
         stmts.append(_new_definition(func_ir, size_var,
                  ir.Const(size, loc=loc), loc))
-
         stmts.append(_new_definition(func_ir, size_tuple_var,
                  ir.Expr.build_tuple(items=[size_var], loc=loc), loc))
 
+        # The general approach is to create an empty array and then fill
+        # the elements in one-by-one from their specificiation.
+
+        # Get the numpy type to pass to empty.
+        nptype = types.DType(dtype)
+
+        # Create a variable to hold the numpy empty function.
         empty_func = ir.Var(scope, mk_unique_var("empty_func"), loc)
         fnty = get_np_ufunc_typ(np.empty)
-        sig = context.resolve_function_type(fnty, (size_typ,), {})
-        typemap[empty_func.name] = fnty #
+        sig = context.resolve_function_type(fnty, (size_typ,), {'dtype':nptype})
+
+        typemap[empty_func.name] = fnty
 
         stmts.append(_new_definition(func_ir, empty_func,
                          ir.Global('empty', np.empty, loc=loc), loc))
 
-        empty_call = ir.Expr.call(empty_func, [size_var], {}, loc=loc)
-        calltypes[empty_call] = typing.signature(array_typ, size_typ)
+        # We pass two arguments to empty, first the size tuple and second
+        # the dtype of the new array.  Here, we created typ_var which is
+        # the dtype argument of the new array.  typ_var in turn is created
+        # by getattr of the dtype string on the numpy module.
+
+        # Create var for numpy module.
+        g_np_var = ir.Var(scope, mk_unique_var("$np_g_var"), loc)
+        typemap[g_np_var.name] = types.misc.Module(np)
+        g_np = ir.Global('np', np, loc)
+        stmts.append(_new_definition(func_ir, g_np_var, g_np, loc))
+
+        # Create var for result of numpy.<dtype>.
+        typ_var = ir.Var(scope, mk_unique_var("$np_typ_var"), loc)
+        typemap[typ_var.name] = nptype
+        dtype_str = str(dtype)
+        if dtype_str == 'bool':
+            dtype_str = 'bool_'
+        # Get dtype attribute of numpy module.
+        np_typ_getattr = ir.Expr.getattr(g_np_var, dtype_str, loc)
+        stmts.append(_new_definition(func_ir, typ_var, np_typ_getattr, loc))
+
+        # Create the call to numpy.empty passing the size tuple and dtype var.
+        empty_call = ir.Expr.call(empty_func, [size_var, typ_var], {}, loc=loc)
+        calltypes[empty_call] = typing.signature(array_typ, size_typ, nptype)
         stmts.append(_new_definition(func_ir, array_var, empty_call, loc))
 
+        # Fill in the new empty array one-by-one.
         for i in range(size):
             index_var = ir.Var(scope, mk_unique_var("index"), loc)
             index_typ = types.intp
diff --git a/numba/npyufunc/parfor.py b/numba/npyufunc/parfor.py
--- a/numba/npyufunc/parfor.py
+++ b/numba/npyufunc/parfor.py
@@ -15,7 +15,7 @@
                             remove_dels, legalize_names, mk_unique_var,
                             rename_labels, get_name_var_table, visit_vars_inner,
                             get_definition, guard, find_callname,
-                            get_call_table, is_pure,
+                            get_call_table, is_pure, get_np_ufunc_typ,
                             get_unused_var_name, find_potential_aliases)
 from numba.analysis import (compute_use_defs, compute_live_map,
                             compute_dead_maps, compute_cfg_from_blocks)
@@ -28,6 +28,8 @@
 
 
 def _lower_parfor_parallel(lowerer, parfor):
+    from .parallel import get_thread_count
+
     """Lowerer that handles LLVM code generation for parfor.
     This function lowers a parfor IR node to LLVM.
     The general approach is as follows:
@@ -44,7 +46,10 @@ def _lower_parfor_parallel(lowerer, parfor):
     targetctx = lowerer.context
     # We copy the typemap here because for race condition variable we'll
     # update their type to array so they can be updated by the gufunc.
-    typemap = copy.copy(lowerer.fndesc.typemap)
+    orig_typemap = lowerer.fndesc.typemap
+    # replace original typemap with copy and restore the original at the end.
+    lowerer.fndesc.typemap = copy.copy(orig_typemap)
+    typemap = lowerer.fndesc.typemap
     varmap = lowerer.varmap
 
     if config.DEBUG_ARRAY_OPT:
@@ -84,6 +89,136 @@ def _lower_parfor_parallel(lowerer, parfor):
         parfor, parfor.params)
     parfor_redvars, parfor_reddict = numba.parfor.get_parfor_reductions(
         parfor, parfor.params, lowerer.fndesc.calltypes)
+
+    # init reduction array allocation here.
+    nredvars = len(parfor_redvars)
+    redarrs = {}
+    if nredvars > 0:
+        # reduction arrays outer dimension equal to thread count
+        thread_count = get_thread_count()
+        scope = parfor.init_block.scope
+        loc = parfor.init_block.loc
+
+        # For each reduction variable...
+        for i in range(nredvars):
+            redvar_typ = lowerer.fndesc.typemap[parfor_redvars[i]]
+            redvar = ir.Var(scope, parfor_redvars[i], loc)
+            redarrvar_typ = redtyp_to_redarraytype(redvar_typ)
+            reddtype = redarrvar_typ.dtype
+            if config.DEBUG_ARRAY_OPT:
+                print("redvar_typ", redvar_typ, redarrvar_typ, reddtype, types.DType(reddtype))
+
+            # If this is reduction over an array,
+            # the reduction array has just one added per-worker dimension.
+            if isinstance(redvar_typ, types.npytypes.Array):
+                redarrdim = redvar_typ.ndim + 1
+            else:
+                redarrdim = 1
+
+            # Reduction array is created and initialized to the initial reduction value.
+
+            # First create a var for the numpy empty ufunc.
+            empty_func = ir.Var(scope, mk_unique_var("empty_func"), loc)
+            ff_fnty = get_np_ufunc_typ(np.empty)
+            ff_sig = typingctx.resolve_function_type(ff_fnty,
+                                            (types.UniTuple(types.intp, redarrdim),
+                                             types.DType(reddtype)), {})
+            typemap[empty_func.name] = ff_fnty
+            empty_assign = ir.Assign(ir.Global("empty", np.empty, loc=loc), empty_func, loc)
+            lowerer.lower_inst(empty_assign)
+
+            # Create var for outer dimension size of reduction array equal to number of threads.
+            num_threads_var = ir.Var(scope, mk_unique_var("num_threads"), loc)
+            num_threads_assign = ir.Assign(ir.Const(thread_count, loc), num_threads_var, loc)
+            typemap[num_threads_var.name] = types.intp
+            lowerer.lower_inst(num_threads_assign)
+
+            # Empty call takes tuple of sizes.  Create here and fill in outer dimension (num threads).
+            size_var = ir.Var(scope, mk_unique_var("tuple_size_var"), loc)
+            typemap[size_var.name] = types.UniTuple(types.intp, redarrdim)
+            size_var_list = [num_threads_var]
+
+            # If this is a reduction over an array...
+            if isinstance(redvar_typ, types.npytypes.Array):
+                # Add code to get the shape of the array being reduced over.
+                redshape_var = ir.Var(scope, mk_unique_var("redarr_shape"), loc)
+                typemap[redshape_var.name] = types.UniTuple(types.intp, redvar_typ.ndim)
+                redshape_getattr = ir.Expr.getattr(redvar, "shape", loc)
+                redshape_assign = ir.Assign(redshape_getattr, redshape_var, loc)
+                lowerer.lower_inst(redshape_assign)
+
+                # Add the dimension sizes of the array being reduced over to the tuple of sizes pass to empty.
+                for j in range(redvar_typ.ndim):
+                    onedimvar = ir.Var(scope, mk_unique_var("redshapeonedim"), loc)
+                    onedimgetitem = ir.Expr.static_getitem(redshape_var, j, None, loc)
+                    typemap[onedimvar.name] = types.intp
+                    onedimassign = ir.Assign(onedimgetitem, onedimvar, loc)
+                    lowerer.lower_inst(onedimassign)
+                    size_var_list += [onedimvar]
+
+            size_call = ir.Expr.build_tuple(size_var_list, loc)
+            size_assign = ir.Assign(size_call, size_var, loc)
+            lowerer.lower_inst(size_assign)
+
+            # Add call to empty passing the size var tuple.
+            empty_call = ir.Expr.call(empty_func, [size_var], {}, loc=loc)
+            redarr_var = ir.Var(scope, mk_unique_var("redarr"), loc)
+            typemap[redarr_var.name] = redarrvar_typ
+            empty_call_assign = ir.Assign(empty_call, redarr_var, loc)
+            lowerer.fndesc.calltypes[empty_call] = ff_sig
+            lowerer.lower_inst(empty_call_assign)
+
+            # Remember mapping of original reduction array to the newly created per-worker reduction array.
+            redarrs[redvar.name] = redarr_var
+
+            init_val = parfor_reddict[parfor_redvars[i]][0]
+            if init_val != None:
+                if isinstance(redvar_typ, types.npytypes.Array):
+                    # Create an array of identity values for the reduction.
+                    # First, create a variable for np.full.
+                    full_func = ir.Var(scope, mk_unique_var("full_func"), loc)
+                    full_fnty = get_np_ufunc_typ(np.full)
+                    full_sig = typingctx.resolve_function_type(full_fnty,
+                               (types.UniTuple(types.intp, redvar_typ.ndim),
+                                reddtype,
+                                types.DType(reddtype)), {})
+                    typemap[full_func.name] = full_fnty
+                    full_assign = ir.Assign(ir.Global("full", np.full, loc=loc), full_func, loc)
+                    lowerer.lower_inst(full_assign)
+
+                    # Then create a var with the identify value.
+                    init_val_var = ir.Var(scope, mk_unique_var("init_val"), loc)
+                    init_val_assign = ir.Assign(ir.Const(init_val, loc), init_val_var, loc)
+                    typemap[init_val_var.name] = reddtype
+                    lowerer.lower_inst(init_val_assign)
+
+                    # Then, call np.full with the shape of the reduction array and the identity value.
+                    full_call = ir.Expr.call(full_func, [redshape_var, init_val_var], {}, loc=loc)
+                    lowerer.fndesc.calltypes[full_call] = full_sig
+                    redtoset = ir.Var(scope, mk_unique_var("redtoset"), loc)
+                    redtoset_assign = ir.Assign(full_call, redtoset, loc)
+                    typemap[redtoset.name] = redvar_typ
+                    lowerer.lower_inst(redtoset_assign)
+                else:
+                    redtoset = ir.Var(scope, mk_unique_var("redtoset"), loc)
+                    redtoset_assign = ir.Assign(ir.Const(init_val, loc), redtoset, loc)
+                    typemap[redtoset.name] = reddtype
+                    lowerer.lower_inst(redtoset_assign)
+            else:
+                redtoset = redvar
+
+            # For each thread, initialize the per-worker reduction array to the current reduction array value.
+            for j in range(get_thread_count()):
+                index_var = ir.Var(scope, mk_unique_var("index_var"), loc)
+                index_var_assign = ir.Assign(ir.Const(j, loc), index_var, loc)
+                typemap[index_var.name] = types.uintp
+                lowerer.lower_inst(index_var_assign)
+
+                redsetitem = ir.SetItem(redarr_var, index_var, redtoset, loc)
+                lowerer.fndesc.calltypes[redsetitem] = signature(types.none,
+                        typemap[redarr_var.name], typemap[index_var.name], redvar_typ)
+                lowerer.lower_inst(redsetitem)
+
     # compile parfor body as a separate function to be used with GUFuncWrapper
     flags = copy.copy(parfor.flags)
     flags.set('error_model', 'numpy')
@@ -93,7 +228,7 @@ def _lower_parfor_parallel(lowerer, parfor):
     for l in parfor.loop_nests[1:]:
         assert typemap[l.index_variable.name] == index_var_typ
     numba.parfor.sequential_parfor_lowering = True
-    func, func_args, func_sig = _create_gufunc_for_parfor_body(
+    func, func_args, func_sig, redargstartdim, func_arg_types = _create_gufunc_for_parfor_body(
         lowerer, parfor, typemap, typingctx, targetctx, flags, {},
         bool(alias_map), index_var_typ, parfor.races)
     numba.parfor.sequential_parfor_lowering = False
@@ -103,16 +238,20 @@ def _lower_parfor_parallel(lowerer, parfor):
     num_reductions = len(parfor_redvars)
     num_inputs = len(func_args) - len(parfor_output_arrays) - num_reductions
     if config.DEBUG_ARRAY_OPT:
+        print("func_args = ", func_args)
         print("num_inputs = ", num_inputs)
         print("parfor_outputs = ", parfor_output_arrays)
         print("parfor_redvars = ", parfor_redvars)
+        print("num_reductions = ", num_reductions)
     gu_signature = _create_shape_signature(
         parfor.get_shape_classes,
         num_inputs,
         num_reductions,
         func_args,
+        redargstartdim,
         func_sig,
-        parfor.races)
+        parfor.races,
+        typemap)
     if config.DEBUG_ARRAY_OPT:
         print("gu_signature = ", gu_signature)
 
@@ -127,15 +266,117 @@ def _lower_parfor_parallel(lowerer, parfor):
         gu_signature,
         func_sig,
         func_args,
+        func_arg_types,
         loop_ranges,
         parfor_redvars,
         parfor_reddict,
+        redarrs,
         parfor.init_block,
         index_var_typ,
         parfor.races)
     if config.DEBUG_ARRAY_OPT:
         sys.stdout.flush()
 
+    if nredvars > 0:
+        # Perform the final reduction across the reduction array created above.
+        thread_count = get_thread_count()
+        scope = parfor.init_block.scope
+        loc = parfor.init_block.loc
+
+        # For each reduction variable...
+        for i in range(nredvars):
+            name = parfor_redvars[i]
+            redarr = redarrs[name]
+            redvar_typ = lowerer.fndesc.typemap[name]
+
+            if config.DEBUG_ARRAY_OPT_RUNTIME:
+                res_print_str = "res_print"
+                strconsttyp = types.Const(res_print_str)
+                lhs = ir.Var(scope, mk_unique_var("str_const"), loc)
+                assign_lhs = ir.Assign(value=ir.Const(value=res_print_str, loc=loc),
+                                               target=lhs, loc=loc)
+                typemap[lhs.name] = strconsttyp
+                lowerer.lower_inst(assign_lhs)
+
+                res_print = ir.Print(args=[lhs, redarr], vararg=None, loc=loc)
+                lowerer.fndesc.calltypes[res_print] = signature(types.none,
+                                                         typemap[lhs.name],
+                                                         typemap[redarr.name])
+                print("res_print", res_print)
+                lowerer.lower_inst(res_print)
+
+            # For each element in the reduction array created above.
+            for j in range(get_thread_count()):
+                # Create index var to access that element.
+                index_var = ir.Var(scope, mk_unique_var("index_var"), loc)
+                index_var_assign = ir.Assign(ir.Const(j, loc), index_var, loc)
+                typemap[index_var.name] = types.uintp
+                lowerer.lower_inst(index_var_assign)
+
+                # Read that element from the array into oneelem.
+                oneelem = ir.Var(scope, mk_unique_var("redelem"), loc)
+                oneelemgetitem = ir.Expr.getitem(redarr, index_var, loc)
+                typemap[oneelem.name] = redvar_typ
+                lowerer.fndesc.calltypes[oneelemgetitem] = signature(redvar_typ,
+                        typemap[redarr.name], typemap[index_var.name])
+                oneelemassign = ir.Assign(oneelemgetitem, oneelem, loc)
+                lowerer.lower_inst(oneelemassign)
+
+                init_var = ir.Var(scope, name+"#init", loc)
+                init_assign = ir.Assign(oneelem, init_var, loc)
+                if name+"#init" not in typemap:
+                    typemap[init_var.name] = redvar_typ
+                lowerer.lower_inst(init_assign)
+
+                if config.DEBUG_ARRAY_OPT_RUNTIME:
+                    res_print_str = "one_res_print"
+                    strconsttyp = types.Const(res_print_str)
+                    lhs = ir.Var(scope, mk_unique_var("str_const"), loc)
+                    assign_lhs = ir.Assign(value=ir.Const(value=res_print_str, loc=loc),
+                                               target=lhs, loc=loc)
+                    typemap[lhs.name] = strconsttyp
+                    lowerer.lower_inst(assign_lhs)
+
+                    res_print = ir.Print(args=[lhs, index_var, oneelem, init_var],
+                                         vararg=None, loc=loc)
+                    lowerer.fndesc.calltypes[res_print] = signature(types.none,
+                                                             typemap[lhs.name],
+                                                             typemap[index_var.name],
+                                                             typemap[oneelem.name],
+                                                             typemap[init_var.name])
+                    print("res_print", res_print)
+                    lowerer.lower_inst(res_print)
+
+                # generate code for combining reduction variable with thread output
+                for inst in parfor_reddict[name][1]:
+                    # If we have a case where a parfor body has an array reduction like A += B
+                    # and A and B have different data types then the reduction in the parallel
+                    # region will operate on those differeing types.  However, here, after the
+                    # parallel region, we are summing across the reduction array and that is
+                    # guaranteed to have the same data type so we need to change the reduction
+                    # nodes so that the right-hand sides have a type equal to the reduction-type
+                    # and therefore the left-hand side.
+                    if isinstance(inst, ir.Assign):
+                        rhs = inst.value
+                        # We probably need to generalize this since it only does substitutions in
+                        # inplace_ginops.
+                        if isinstance(rhs, ir.Expr) and rhs.op == 'inplace_binop' and rhs.rhs.name == init_var.name:
+                            # Get calltype of rhs.
+                            ct = lowerer.fndesc.calltypes[rhs]
+                            assert(len(ct.args) == 2)
+                            # Create new arg types replace the second arg type with the reduction var type.
+                            ctargs = (ct.args[0], redvar_typ)
+                            # Update the signature of the call.
+                            ct = ct.replace(args=ctargs)
+                            # Remove so we can re-insrt since calltypes is unique dict.
+                            lowerer.fndesc.calltypes.pop(rhs)
+                            # Add calltype back in for the expr with updated signature.
+                            lowerer.fndesc.calltypes[rhs] = ct
+                    lowerer.lower_inst(inst)
+
+    # Restore the original typemap of the function that was replaced temporarily at the
+    # Beginning of this function.
+    lowerer.fndesc.typemap = orig_typemap
 
 # A work-around to prevent circular imports
 lowering.lower_extensions[parfor.Parfor] = _lower_parfor_parallel
@@ -146,18 +387,20 @@ def _create_shape_signature(
         num_inputs,
         num_reductions,
         args,
+        redargstartdim,
         func_sig,
-        races):
+        races,
+        typemap):
     '''Create shape signature for GUFunc
     '''
     if config.DEBUG_ARRAY_OPT:
-        print("_create_shape_signature", num_inputs, num_reductions, args, func_sig)
+        print("_create_shape_signature", num_inputs, num_reductions, args, redargstartdim)
         for i in args[1:]:
-            print("argument", i, type(i), get_shape_classes(i))
+            print("argument", i, type(i), get_shape_classes(i, typemap=typemap))
 
     num_inouts = len(args) - num_reductions
     # maximum class number for array shapes
-    classes = [get_shape_classes(var) if var not in races else (-1,) for var in args[1:]]
+    classes = [get_shape_classes(var, typemap=typemap) if var not in races else (-1,) for var in args[1:]]
     class_set = set()
     for _class in classes:
         if _class:
@@ -187,20 +430,19 @@ def bump_alpha(c, class_map):
     gu_sout = []
     count = 0
     syms_sin = ()
-    for cls in classes:
-        # print("create_shape_signature: var = ", var, " typ = ", typ)
+    if config.DEBUG_ARRAY_OPT:
+        print("args", args)
+        print("classes", classes)
+    for cls, arg in zip(classes, args):
         count = count + 1
         if cls:
             dim_syms = tuple(bump_alpha(c, class_map) for c in cls)
         else:
             dim_syms = ()
         if (count > num_inouts):
-            # assume all reduction vars are scalar
-            gu_sout.append(())
-        elif count > num_inputs and all([s in syms_sin for s in dim_syms]):
-            # only when dim_syms are found in gu_sin, we consider this as
-            # output
-            gu_sout.append(dim_syms)
+            # Strip the first symbol corresponding to the number of workers
+            # so that guvectorize will parallelize across the reduction.
+            gu_sin.append(dim_syms[redargstartdim[arg]:])
         else:
             gu_sin.append(dim_syms)
             syms_sin += dim_syms
@@ -329,6 +571,29 @@ def hoist(parfor_params, loop_body, typemap, wrapped_blocks):
         block.body = new_block
     return hoisted
 
+def redtyp_is_scalar(redtype):
+    return not isinstance(redtype, types.npytypes.Array)
+
+def redtyp_to_redarraytype(redtyp):
+    """Go from a reducation variable type to a reduction array type used to hold
+       per-worker results.
+    """
+    redarrdim = 1
+    # If the reduction type is an array then allocate reduction array with ndim+1 dimensions.
+    if isinstance(redtyp, types.npytypes.Array):
+        redarrdim += redtyp.ndim
+        # We don't create array of array but multi-dimensional reduciton array with same dtype.
+        redtyp = redtyp.dtype
+    return types.npytypes.Array(redtyp, redarrdim, "C")
+
+def redarraytype_to_sig(redarraytyp):
+    """Given a reduction array type, find the type of the reduction argument to the gufunc.
+       Scalar and 1D array reduction both end up with 1D gufunc param type since scalars have to
+       be passed as arrays.
+    """
+    assert isinstance(redarraytyp, types.npytypes.Array)
+    return types.npytypes.Array(redarraytyp.dtype, max(1, redarraytyp.ndim - 1), redarraytyp.layout)
+
 def _create_gufunc_for_parfor_body(
         lowerer,
         parfor,
@@ -369,6 +634,7 @@ def _create_gufunc_for_parfor_body(
     # Get just the outputs of the parfor.
     parfor_outputs = numba.parfor.get_parfor_outputs(parfor, parfor_params)
     # Get all parfor reduction vars, and operators.
+    typemap = lowerer.fndesc.typemap
     parfor_redvars, parfor_reddict = numba.parfor.get_parfor_reductions(
         parfor, parfor_params, lowerer.fndesc.calltypes)
     # Compute just the parfor inputs as a set difference.
@@ -397,10 +663,13 @@ def _create_gufunc_for_parfor_body(
     # Reduction variables are represented as arrays, so they go under
     # different names.
     parfor_redarrs = []
+    parfor_red_arg_types = []
     for var in parfor_redvars:
         arr = var + "_arr"
         parfor_redarrs.append(arr)
-        typemap[arr] = types.npytypes.Array(typemap[var], 1, "C")
+        redarraytype = redtyp_to_redarraytype(typemap[var])
+        typemap[arr] = redarraytype_to_sig(redarraytype)
+        parfor_red_arg_types.append(redarraytype)
 
     # Reorder all the params so that inputs go first then outputs.
     parfor_params = parfor_inputs + parfor_outputs + parfor_redarrs
@@ -440,10 +709,8 @@ def _create_gufunc_for_parfor_body(
 
     # Get the types of each parameter.
     param_types = [typemap[v] for v in parfor_params]
-    # if config.DEBUG_ARRAY_OPT==1:
-    #    param_types_dict = { v:typemap[v] for v in parfor_params }
-    #    print("param_types_dict = ", param_types_dict, " ", type(param_types_dict))
-    #    print("param_types = ", param_types, " ", type(param_types))
+    # Calculate types of args passed to gufunc.
+    func_arg_types = [typemap[v] for v in (parfor_inputs + parfor_outputs)] + parfor_red_arg_types
 
     # Replace illegal parameter names in the loop body with legal ones.
     replace_var_names(loop_body, param_dict)
@@ -456,7 +723,9 @@ def _create_gufunc_for_parfor_body(
     parfor_params = []
     ascontig = False
     for pindex in range(len(parfor_params_orig)):
-        if ascontig and pindex < len(parfor_inputs) and isinstance(param_types[pindex], types.npytypes.Array):
+        if (ascontig and
+            pindex < len(parfor_inputs) and
+            isinstance(param_types[pindex], types.npytypes.Array)):
             parfor_params.append(parfor_params_orig[pindex]+"param")
         else:
             parfor_params.append(parfor_params_orig[pindex])
@@ -494,8 +763,15 @@ def _create_gufunc_for_parfor_body(
 
     # Add initialization of reduction variables
     for arr, var in zip(parfor_redarrs, parfor_redvars):
-        gufunc_txt += "    " + param_dict[var] + \
-            "=" + param_dict[arr] + "[0]\n"
+        # If reduction variable is a scalar then save current value to
+        # temp and accumulate on that temp to prevent false sharing.
+        if redtyp_is_scalar(typemap[var]):
+            gufunc_txt += "    " + param_dict[var] + \
+                 "=" + param_dict[arr] + "[0]\n"
+        else:
+            # The reduction variable is an array so np.copy it to a temp.
+            gufunc_txt += "    " + param_dict[var] + \
+                 "=np.copy(" + param_dict[arr] + ")\n"
 
     # For each dimension of the parfor, create a for loop in the generated gufunc function.
     # Iterate across the proper values extracted from the schedule.
@@ -528,9 +804,18 @@ def _create_gufunc_for_parfor_body(
         gufunc_txt += "    "
     gufunc_txt += sentinel_name + " = 0\n"
     # Add assignments of reduction variables (for returning the value)
+    redargstartdim = {}
     for arr, var in zip(parfor_redarrs, parfor_redvars):
-        gufunc_txt += "    " + param_dict[arr] + \
-            "[0] = " + param_dict[var] + "\n"
+        # After the gufunc loops, copy the accumulated temp value back to reduction array.
+        if redtyp_is_scalar(typemap[var]):
+            gufunc_txt += "    " + param_dict[arr] + \
+                "[0] = " + param_dict[var] + "\n"
+            redargstartdim[arr] = 1
+        else:
+            # After the gufunc loops, copy the accumulated temp array back to reduction array with ":"
+            gufunc_txt += "    " + param_dict[arr] + \
+                "[:] = " + param_dict[var] + "[:]\n"
+            redargstartdim[arr] = 0
     gufunc_txt += "    return None\n"
 
     if config.DEBUG_ARRAY_OPT:
@@ -702,7 +987,7 @@ def _create_gufunc_for_parfor_body(
     if config.DEBUG_ARRAY_OPT:
         print("kernel_sig = ", kernel_sig)
 
-    return kernel_func, parfor_args, kernel_sig
+    return kernel_func, parfor_args, kernel_sig, redargstartdim, func_arg_types
 
 def replace_var_with_array_in_block(vars, block, typemap, calltypes):
     new_block = []
@@ -737,8 +1022,8 @@ def replace_var_with_array(vars, loop_body, typemap, calltypes):
         typemap.pop(v, None)
         typemap[v] = types.npytypes.Array(el_typ, 1, "C")
 
-def call_parallel_gufunc(lowerer, cres, gu_signature, outer_sig, expr_args,
-                         loop_ranges, redvars, reddict, init_block, index_var_typ, races):
+def call_parallel_gufunc(lowerer, cres, gu_signature, outer_sig, expr_args, expr_arg_types,
+                         loop_ranges, redvars, reddict, redarrdict, init_block, index_var_typ, races):
     '''
     Adds the call to the gufunc function from the main function.
     '''
@@ -755,6 +1040,9 @@ def call_parallel_gufunc(lowerer, cres, gu_signature, outer_sig, expr_args,
         print("outer_sig = ", outer_sig.args, outer_sig.return_type,
               outer_sig.recvr, outer_sig.pysig)
         print("loop_ranges = ", loop_ranges)
+        print("expr_args", expr_args)
+        print("expr_arg_types", expr_arg_types)
+        print("gu_signature", gu_signature)
 
     # Build the wrapper for GUFunc
     args, return_type = sigutils.normalize_signature(outer_sig)
@@ -868,27 +1156,11 @@ def load_range(v):
                 types.uintp, get_thread_count()), sched, context.get_constant(
                     types.intp, debug_flag)])
 
-    # init reduction array allocation here.
+    # Get the LLVM vars for the Numba IR reduction array vars.
+    redarrs = [lowerer.loadvar(x.name) for x in list(redarrdict.values())]
+
     nredvars = len(redvars)
     ninouts = len(expr_args) - nredvars
-    redarrs = []
-    for i in range(nredvars):
-        redvar_typ = lowerer.fndesc.typemap[redvars[i]]
-        # we need to use the default initial value instead of existing value in
-        # redvar if available
-        init_val = reddict[redvars[i]][0]
-        if init_val != None:
-            val = context.get_constant(redvar_typ, init_val)
-        else:
-            val = lowerer.loadvar(redvars[i])
-        typ = context.get_value_type(redvar_typ)
-        size = get_thread_count()
-        arr = cgutils.alloca_once(builder, typ,
-                                  size=context.get_constant(types.uintp, size))
-        redarrs.append(arr)
-        for j in range(size):
-            dst = builder.gep(arr, [context.get_constant(types.uintp, j)])
-            builder.store(val, dst)
 
     if config.DEBUG_ARRAY_OPT:
         for i in range(get_thread_count()):
@@ -902,6 +1174,7 @@ def load_range(v):
                                     types.intp, i * num_dim * 2 + j)])))
             cgutils.printf(builder, "\n")
 
+    # ----------------------------------------------------------------------------
     # Prepare arguments: args, shapes, steps, data
     all_args = [lowerer.loadvar(x) for x in expr_args[:ninouts]] + redarrs
     num_args = len(all_args)
@@ -917,15 +1190,23 @@ def load_range(v):
     # sched goes first
     builder.store(builder.bitcast(sched, byte_ptr_t), args)
     array_strides.append(context.get_constant(types.intp, sizeof_intp))
+    red_shapes = {}
     rv_to_arg_dict = {}
     # followed by other arguments
     for i in range(num_args):
         arg = all_args[i]
         var = expr_args[i]
-        aty = outer_sig.args[i + 1]  # skip first argument sched
+        aty = expr_arg_types[i]
         dst = builder.gep(args, [context.get_constant(types.intp, i + 1)])
         if i >= ninouts:  # reduction variables
-            builder.store(builder.bitcast(arg, byte_ptr_t), dst)
+            ary = context.make_array(aty)(context, builder, arg)
+            strides = cgutils.unpack_tuple(builder, ary.strides, aty.ndim)
+            ary_shapes = cgutils.unpack_tuple(builder, ary.shape, aty.ndim)
+            # Start from 1 because we skip the first dimension of length num_threads just like sched.
+            for j in range(1, len(strides)):
+                array_strides.append(strides[j])
+            red_shapes[i] = ary_shapes[1:]
+            builder.store(builder.bitcast(ary.data, byte_ptr_t), dst)
         elif isinstance(aty, types.ArrayCompatible):
             if var in races:
                 typ = context.get_data_type(
@@ -957,16 +1238,25 @@ def load_range(v):
                 ptr = cgutils.alloca_once(builder, typ)
             builder.store(builder.bitcast(ptr, byte_ptr_t), dst)
 
+    # ----------------------------------------------------------------------------
     # Next, we prepare the individual dimension info recorded in gu_signature
     sig_dim_dict = {}
     occurances = []
     occurances = [sched_sig[0]]
     sig_dim_dict[sched_sig[0]] = context.get_constant(types.intp, 2 * num_dim)
-    for var, arg, aty, gu_sig in zip(expr_args[:ninouts], all_args[:ninouts],
-                                     outer_sig.args[1:], sin + sout):
+    assert len(expr_args) == len(all_args)
+    assert len(expr_args) == len(expr_arg_types)
+    assert len(expr_args) == len(sin + sout)
+    assert len(expr_args) == len(outer_sig.args[1:])
+    for var, arg, aty, gu_sig in zip(expr_args, all_args,
+                                     expr_arg_types, sin + sout):
+        if isinstance(aty, types.npytypes.Array):
+            i = aty.ndim - len(gu_sig)
+        else:
+            i = 0
         if config.DEBUG_ARRAY_OPT:
-            print("var = ", var, " gu_sig = ", gu_sig)
-        i = 0
+            print("var =", var, "gu_sig =", gu_sig, "type =", aty, "i =", i)
+
         for dim_sym in gu_sig:
             if config.DEBUG_ARRAY_OPT:
                 print("var = ", var, " type = ", aty)
@@ -984,6 +1274,7 @@ def load_range(v):
                 occurances.append(dim_sym)
             i = i + 1
 
+    # ----------------------------------------------------------------------------
     # Prepare shapes, which is a single number (outer loop size), followed by
     # the size of individual shape variables.
     nshapes = len(sig_dim_dict) + 1
@@ -1002,6 +1293,7 @@ def load_range(v):
                         types.intp, i)]))
         i = i + 1
 
+    # ----------------------------------------------------------------------------
     # Prepare steps for each argument. Note that all steps are counted in
     # bytes.
     num_steps = num_args + 1 + len(array_strides)
@@ -1015,9 +1307,19 @@ def load_range(v):
     for i in range(num_args):
         if i >= ninouts:  # steps for reduction vars are abi_sizeof(typ)
             j = i - ninouts
-            typ = context.get_value_type(lowerer.fndesc.typemap[redvars[j]])
+            # Get the base dtype of the reduction array.
+            redtyp = lowerer.fndesc.typemap[redvars[j]]
+            red_stride = None
+            if isinstance(redtyp, types.npytypes.Array):
+                redtyp = redtyp.dtype
+                red_stride = red_shapes[i]
+            typ = context.get_value_type(redtyp)
             sizeof = context.get_abi_sizeof(typ)
+            # Set stepsize to the size of that dtype.
             stepsize = context.get_constant(types.intp, sizeof)
+            if red_stride != None:
+                for rs in red_stride:
+                    stepsize = builder.mul(stepsize, rs)
         else:
             # steps are strides
             stepsize = zero
@@ -1030,6 +1332,7 @@ def load_range(v):
                     types.intp, 1 + num_args + j)])
         builder.store(array_strides[j], dst)
 
+    # ----------------------------------------------------------------------------
     # prepare data
     data = builder.inttoptr(zero, byte_ptr_t)
 
@@ -1049,25 +1352,3 @@ def load_range(v):
 
     scope = init_block.scope
     loc = init_block.loc
-    calltypes = lowerer.fndesc.calltypes
-    # Accumulate all reduction arrays back to a single value
-    for i in range(get_thread_count()):
-        for name, arr in zip(redvars, redarrs):
-            tmpname = mk_unique_var(name)
-            src = builder.gep(arr, [context.get_constant(types.intp, i)])
-            val = builder.load(src)
-            vty = lowerer.fndesc.typemap[name]
-            lowerer.fndesc.typemap[tmpname] = vty
-            lowerer.storevar(val, tmpname)
-            tmpvar = ir.Var(scope, tmpname, loc)
-            tmp_assign = ir.Assign(tmpvar, ir.Var(scope, name+"#init", loc), loc)
-            if name+"#init" not in lowerer.fndesc.typemap:
-                lowerer.fndesc.typemap[name+"#init"] = vty
-            lowerer.lower_inst(tmp_assign)
-            # generate code for combining reduction variable with thread output
-            for inst in reddict[name][1]:
-                lowerer.lower_inst(inst)
-
-    # TODO: scalar output must be assigned back to corresponding output
-    # variables
-    return
diff --git a/numba/parfor.py b/numba/parfor.py
--- a/numba/parfor.py
+++ b/numba/parfor.py
@@ -483,8 +483,23 @@ def list_vars(self):
 
         return all_uses
 
-    def get_shape_classes(self, var):
-        return self.equiv_set.get_shape_classes(var)
+    def get_shape_classes(self, var, typemap=None):
+        """get the shape classes for a given variable.
+        If a typemap is specified then use it for type resolution
+        """
+        # We get shape classes from the equivalence set but that
+        # keeps its own typemap at a time prior to lowering.  So
+        # if something is added during lowering then we can pass
+        # in a type map to use.  We temporarily replace the
+        # equivalence set typemap, do the work and then restore
+        # the original on the way out.
+        if typemap is not None:
+            save_typemap = self.equiv_set.typemap
+            self.equiv_set.typemap = typemap
+        res = self.equiv_set.get_shape_classes(var)
+        if typemap is not None:
+            self.equiv_set.typemap = save_typemap
+        return res
 
     def dump(self, file=None):
         file = file or sys.stdout


Test patch:
diff --git a/numba/tests/test_parfors.py b/numba/tests/test_parfors.py
--- a/numba/tests/test_parfors.py
+++ b/numba/tests/test_parfors.py
@@ -1188,6 +1188,76 @@ def test_impl(n):
         # as a different name
         self.assertEqual(countArrayAllocs(test_impl, (types.intp,)), 1)
 
+    @skip_unsupported
+    def test_one_d_array_reduction(self):
+        def test_impl(n):
+            result = np.zeros(1, np.int_)
+
+            for i in numba.prange(n):
+                result += np.array([i], np.int_)
+
+            return result
+
+        self.check(test_impl, 100)
+
+    @skip_unsupported
+    def test_two_d_array_reduction(self):
+        def test_impl(n):
+            shp = (13, 17)
+            size = shp[0] * shp[1]
+            result1 = np.zeros(shp, np.int_)
+            tmp = np.arange(size).reshape(shp)
+
+            for i in numba.prange(n):
+                result1 += tmp
+
+            return result1
+
+        self.check(test_impl, 100)
+
+    @skip_unsupported
+    def test_two_d_array_reduction_with_float_sizes(self):
+        # result1 is float32 and tmp is float64.
+        # Tests reduction with differing dtypes.
+        def test_impl(n):
+            shp = (2, 3)
+            result1 = np.zeros(shp, np.float32)
+            tmp = np.array([1.0, 2.0, 3.0, 4.0, 5.0, 6.0]).reshape(shp)
+
+            for i in numba.prange(n):
+                result1 += tmp
+
+            return result1
+
+        self.check(test_impl, 100)
+
+    @skip_unsupported
+    def test_two_d_array_reduction_prod(self):
+        def test_impl(n):
+            shp = (13, 17)
+            result1 = 2 * np.ones(shp, np.int_)
+            tmp = 2 * np.ones_like(result1)
+
+            for i in numba.prange(n):
+                result1 *= tmp
+
+            return result1
+
+        self.check(test_impl, 100)
+
+    @skip_unsupported
+    def test_three_d_array_reduction(self):
+        def test_impl(n):
+            shp = (3, 2, 7)
+            result1 = np.zeros(shp, np.int_)
+
+            for i in numba.prange(n):
+                result1 += np.ones(shp, np.int_)
+
+            return result1
+
+        self.check(test_impl, 100)
+
     @skip_unsupported
     def test_preparfor_canonicalize_kws(self):
         # test canonicalize_array_math typing for calls with kw args