Repository: numba/numba

Issue:
np.var and np.std impls do not handle complex types correctly
```python
import numba as nb
import numpy as np

@nb.njit
def nb_std(x):
    return np.std(x)

@nb.njit
def nb_var(x):
    return np.var(x)

areal = np.arange(10.)
acomplex = areal + 2.j

for x in (areal, acomplex):
    print("testing std:")
    print(np.std(x))
    print(nb_std(x))
    print("testing var")
    print(np.var(x))
    print(nb_var(x))
    print("")
```
gives:
```
$ python test1.py 
testing std:
2.87228132327
2.8722813232690143
testing var
8.25
8.25

testing std:
2.87228132327
(2.8722813232690143+0j)
testing var
8.25
(8.25+0j)
```
the Numba impls incorrectly return complex numbers for complex input.


Discussion:
In fact, it's worse than just typing, they return the wrong results.
```python
import numba as nb
import numpy as np

@nb.njit
def nb_std(x):
    return np.std(x)

@nb.njit
def nb_var(x):
    return np.var(x)

a=np.array([1+2j, 2-4j, -4+5j])

print("testing std:")
print(np.std(a))
print(nb_std(a))
print("testing var")
print(np.var(a))
print(nb_var(a))
```
gives:
```
$ python test1.py 
testing std:
4.57043640027
(2.346188410725049-3.55186024073747j)
testing var
20.8888888889
(-7.111111111111111-16.666666666666668j)
```
I am guessing this is down to using `((x - x.mean()) ** 2).sum()/x.size` where the power should be multiply by conjugate in the complex case https://github.com/numba/numba/blob/master/numba/targets/arraymath.py#L334

Resolving patch:
diff --git a/numba/parfor.py b/numba/parfor.py
--- a/numba/parfor.py
+++ b/numba/parfor.py
@@ -284,7 +284,8 @@ def var_1(in_arr):
             numba.parfor.init_prange()
             ssd = 0
             for i in numba.parfor.internal_prange(len(in_arr)):
-                ssd += (in_arr[i] - m) ** 2
+                val = in_arr[i] - m
+                ssd += np.real(val * np.conj(val))
             return ssd / len(in_arr)
     else:
         def var_1(in_arr):
@@ -294,7 +295,8 @@ def var_1(in_arr):
             numba.parfor.init_prange()
             ssd = 0
             for i in numba.pndindex(in_arr.shape):
-                ssd += (in_arr[i] - m) ** 2
+                val = in_arr[i] - m
+                ssd += np.real(val * np.conj(val))
             return ssd / in_arr.size
     return var_1
 
diff --git a/numba/targets/arraymath.py b/numba/targets/arraymath.py
--- a/numba/targets/arraymath.py
+++ b/numba/targets/arraymath.py
@@ -342,7 +342,8 @@ def array_var_impl(arr):
         # Compute the sum of square diffs
         ssd = 0
         for v in np.nditer(arr):
-            ssd += (v.item() - m) ** 2
+            val = (v.item() - m)
+            ssd +=  np.real(val * np.conj(val))
         return ssd / arr.size
 
     res = context.compile_internal(builder, array_var_impl, sig, args)
@@ -599,7 +600,8 @@ def nanvar_impl(arr):
             for view in np.nditer(arr):
                 v = view.item()
                 if not isnan(v):
-                    ssd += (v.item() - m) ** 2
+                    val = (v.item() - m)
+                    ssd +=  np.real(val * np.conj(val))
                     count += 1
             # np.divide() doesn't raise ZeroDivisionError
             return np.divide(ssd, count)
diff --git a/numba/typing/arraydecl.py b/numba/typing/arraydecl.py
--- a/numba/typing/arraydecl.py
+++ b/numba/typing/arraydecl.py
@@ -646,6 +646,15 @@ def generic_hetero_real(self, args, kws):
         return signature(types.float64, recvr=self.this)
     return signature(self.this.dtype, recvr=self.this)
 
+def generic_hetero_always_real(self, args, kws):
+    assert not args
+    assert not kws
+    if isinstance(self.this.dtype, (types.Integer, types.Boolean)):
+        return signature(types.float64, recvr=self.this)
+    if isinstance(self.this.dtype, types.Complex):
+        return signature(self.this.dtype.underlying_float, recvr=self.this)
+    return signature(self.this.dtype, recvr=self.this)
+
 def generic_index(self, args, kws):
     assert not args
     assert not kws
@@ -674,9 +683,15 @@ def array_attribute_attachment(self, ary):
     install_array_method(fname, generic_expand_cumulative)
 
 # Functions that require integer arrays get promoted to float64 return
-for fName in ["mean", "var", "std"]:
+for fName in ["mean"]:
     install_array_method(fName, generic_hetero_real)
 
+# var and std by definition return in real space and int arrays
+# get promoted to float64 return
+for fName in ["var", "std"]:
+    install_array_method(fName, generic_hetero_always_real)
+
+
 # Functions that return an index (intp)
 install_array_method("argmin", generic_index)
 install_array_method("argmax", generic_index)


Test patch:
diff --git a/numba/tests/test_array_reductions.py b/numba/tests/test_array_reductions.py
--- a/numba/tests/test_array_reductions.py
+++ b/numba/tests/test_array_reductions.py
@@ -146,6 +146,15 @@ def full_test_arrays(dtype):
     if dtype == np.float32:
         array_list += [a / 10 for a in array_list]
 
+    # add imaginary part
+    if dtype == np.complex64:
+        acc = []
+        for a in array_list:
+            tmp = a / 10 + 1j * a / 11
+            tmp[::2] = np.conj(tmp[::2])
+            acc.append(tmp)
+        array_list.extend(acc)
+
     for a in array_list:
         assert a.dtype == np.dtype(dtype)
     return array_list
@@ -600,41 +609,59 @@ def test_mean_npdatetime(self):
     @classmethod
     def install_generated_tests(cls):
         # These form a testing product where each of the combinations are tested
+
+        # these function are tested in real and complex space
         reduction_funcs = [array_sum, array_sum_global,
                            array_prod, array_prod_global,
                            array_mean, array_mean_global,
                            array_var, array_var_global,
                            array_std, array_std_global,
-                           array_min, array_min_global,
-                           array_max, array_max_global,
-                           array_argmin, array_argmin_global,
-                           array_argmax, array_argmax_global,
                            array_all, array_all_global,
                            array_any, array_any_global,
-                           array_nanmax,
-                           array_nanmin,
                            array_nansum,
                            ]
+
+        # these functions only work in real space as no complex comparison
+        # operator is implemented
+        reduction_funcs_rspace = [array_min, array_min_global,
+                                  array_max, array_max_global,
+                                  array_argmin, array_argmin_global,
+                                  array_argmax, array_argmax_global,
+                                  array_nanmax, array_nanmin]
+
         if np_version >= (1, 8):
             reduction_funcs += [array_nanmean, array_nanstd, array_nanvar]
         if np_version >= (1, 10):
             reduction_funcs += [array_nanprod]
 
-        dtypes_to_test = [np.int32, np.float32, np.bool_]
-
-        # Install tests on class
-        for dt in dtypes_to_test:
-            test_arrays = full_test_arrays(dt)
-            for red_func, test_array in product(reduction_funcs, test_arrays):
-                # Create the name for the test function
-                test_name = "test_{0}_{1}_{2}d".format(red_func.__name__, test_array.dtype.name, test_array.ndim)
-
-                def new_test_function(self, redFunc=red_func, testArray=test_array, testName=test_name):
-                    npr, nbr = run_comparative(redFunc, testArray)
-                    self.assertPreciseEqual(npr, nbr, msg=test_name, prec="single")
-
-                # Install it into the class
-                setattr(cls, test_name, new_test_function)
+        dtypes_to_test = [np.int32, np.float32, np.bool_, np.complex64]
+
+        def install_tests(dtypes, funcs):
+            # Install tests on class
+            for dt in dtypes:
+                test_arrays = full_test_arrays(dt)
+                for red_func, test_array in product(funcs, test_arrays):
+                    # Create the name for the test function
+                    test_name = "test_{0}_{1}_{2}d"
+                    test_name = test_name.format(red_func.__name__,
+                                                 test_array.dtype.name,
+                                                 test_array.ndim)
+
+                    def new_test_function(self, redFunc=red_func,
+                                          testArray=test_array,
+                                          testName=test_name):
+                        npr, nbr = run_comparative(redFunc, testArray)
+                        self.assertPreciseEqual(npr, nbr, msg=test_name,
+                                                prec="single")
+
+                    # Install it into the class
+                    setattr(cls, test_name, new_test_function)
+
+        # install tests for reduction functions that only work in real space
+        install_tests(dtypes_to_test[:-1], reduction_funcs)
+
+        # install tests for reduction functions
+        install_tests(dtypes_to_test, reduction_funcs)
 
 
 TestArrayReductions.install_generated_tests()
diff --git a/numba/tests/test_parfors.py b/numba/tests/test_parfors.py
--- a/numba/tests/test_parfors.py
+++ b/numba/tests/test_parfors.py
@@ -814,8 +814,10 @@ def test_impl(A):
         N = 100
         A = np.random.ranf(N)
         B = np.random.randint(10, size=(N, 3))
+        C = A + 1j * A
         self.check(test_impl, A)
         self.check(test_impl, B)
+        self.check(test_impl, C)
         self.assertTrue(countParfors(test_impl, (types.Array(types.float64, 1, 'C'), )) == 2)
         self.assertTrue(countParfors(test_impl, (types.Array(types.float64, 2, 'C'), )) == 2)
 
@@ -826,8 +828,10 @@ def test_impl(A):
         N = 100
         A = np.random.ranf(N)
         B = np.random.randint(10, size=(N, 3))
+        C = A + 1j * A
         self.check(test_impl, A)
         self.check(test_impl, B)
+        self.check(test_impl, C)
         self.assertTrue(countParfors(test_impl, (types.Array(types.float64, 1, 'C'), )) == 2)
         self.assertTrue(countParfors(test_impl, (types.Array(types.float64, 2, 'C'), )) == 2)