Repository: numpy/numpy
Pull request: ENH: add dtype option to numpy.lib.function_base.cov and corrcoef
I tried to implement the changes suggested in [Issue #17395](https://github.com/numpy/numpy/issues/17395#issue-711108059), by adding a dtype keyword to the cov and corrcoef functions in numpy/lib/function_base.py. I also wrote tests for the changes.

Closes #17395 

Problem statement:
dtype control for corrcoef
## Feature

I am computing a cross-correlation matrix of 150K * 150K matrix of float32.

Numpy insists on making the results dtype be float64. This causes the amount of memory to be "too much".

I would debate that default, but changing it would probably be too disruptive (not backward compatible).

However, how about a dtype parameter for corrcoef?

In general I would have liked to see a dtype parameter "everywhere" to override the float64 default...

Hints from issue discussion:

Commit message: add pass dtype argument to von inside corrcoef

Patch to review:
diff --git a/numpy/lib/function_base.py b/numpy/lib/function_base.py
index c7ddbdb8de67..d54ac1b2592c 100644
--- a/numpy/lib/function_base.py
+++ b/numpy/lib/function_base.py
@@ -2268,13 +2268,13 @@ def _vectorize_call_with_signature(self, func, args):
 
 
 def _cov_dispatcher(m, y=None, rowvar=None, bias=None, ddof=None,
-                    fweights=None, aweights=None):
+                    fweights=None, aweights=None, dtype=None):
     return (m, y, fweights, aweights)
 
 
 @array_function_dispatch(_cov_dispatcher)
 def cov(m, y=None, rowvar=True, bias=False, ddof=None, fweights=None,
-        aweights=None):
+        aweights=None, dtype=None):
     """
     Estimate a covariance matrix, given data and weights.
 
@@ -2400,13 +2400,15 @@ def cov(m, y=None, rowvar=True, bias=False, ddof=None, fweights=None,
     if m.ndim > 2:
         raise ValueError("m has more than 2 dimensions")
 
-    if y is None:
-        dtype = np.result_type(m, np.float64)
-    else:
-        y = np.asarray(y)
-        if y.ndim > 2:
-            raise ValueError("y has more than 2 dimensions")
-        dtype = np.result_type(m, y, np.float64)
+    if dtype is None:
+        dtype = np.float64
+        if y is None:
+            dtype = np.result_type(m, dtype)
+        else:
+            y = np.asarray(y)
+            if y.ndim > 2:
+                raise ValueError("y has more than 2 dimensions")
+            dtype = np.result_type(m, y, dtype)
 
     X = array(m, ndmin=2, dtype=dtype)
     if not rowvar and X.shape[0] != 1:
@@ -2486,12 +2488,14 @@ def cov(m, y=None, rowvar=True, bias=False, ddof=None, fweights=None,
     return c.squeeze()
 
 
-def _corrcoef_dispatcher(x, y=None, rowvar=None, bias=None, ddof=None):
+def _corrcoef_dispatcher(x, y=None, rowvar=None, bias=None, ddof=None,
+                         dtype=None):
     return (x, y)
 
 
 @array_function_dispatch(_corrcoef_dispatcher)
-def corrcoef(x, y=None, rowvar=True, bias=np._NoValue, ddof=np._NoValue):
+def corrcoef(x, y=None, rowvar=True, bias=np._NoValue, ddof=np._NoValue,
+             dtype=None):
     """
     Return Pearson product-moment correlation coefficients.
 
@@ -2616,7 +2620,7 @@ def corrcoef(x, y=None, rowvar=True, bias=np._NoValue, ddof=np._NoValue):
         # 2015-03-15, 1.10
         warnings.warn('bias and ddof have no effect and are deprecated',
                       DeprecationWarning, stacklevel=3)
-    c = cov(x, y, rowvar)
+    c = cov(x, y, rowvar, dtype=dtype)
     try:
         d = diag(c)
     except ValueError:
diff --git a/numpy/lib/tests/test_function_base.py b/numpy/lib/tests/test_function_base.py
index 7bddb941c5c8..4763c87fa935 100644
--- a/numpy/lib/tests/test_function_base.py
+++ b/numpy/lib/tests/test_function_base.py
@@ -2023,6 +2023,15 @@ def test_extreme(self):
         assert_array_almost_equal(c, np.array([[1., -1.], [-1., 1.]]))
         assert_(np.all(np.abs(c) <= 1.0))
 
+    @pytest.mark.parametrize("test_type", [getattr(np, dtype, None) for dtype \
+    in ['float32', 'float64', 'float96', 'float128']])
+    def test_cov_dtype(self, test_type):
+        if test_type is None:
+            pytest.skip("Data type not available")
+        cast_A = self.A.astype(test_type)
+        res = corrcoef(cast_A, dtype=test_type)
+        assert test_type == res.dtype
+
 
 class TestCov:
     x1 = np.array([[0, 2], [1, 1], [2, 0]]).T
@@ -2123,6 +2132,15 @@ def test_unit_fweights_and_aweights(self):
                             aweights=self.unit_weights),
                         self.res1)
 
+    @pytest.mark.parametrize("test_type", [getattr(np, dtype, None) for dtype \
+        in ['float32', 'float64', 'float96', 'float128']])
+    def test_cov_dtype(self, test_type):
+        if test_type is None:
+            pytest.skip("Data type not available")
+        cast_x1 = self.x1.astype(test_type)
+        res = cov(cast_x1, dtype=test_type)
+        assert test_type == res.dtype
+
 
 class Test_I0:
 


Code review comments:
diff_hunk: @@ -2400,13 +2400,15 @@ def cov(m, y=None, rowvar=True, bias=False, ddof=None, fweights=None,
     if m.ndim > 2:
         raise ValueError("m has more than 2 dimensions")
 
-    if y is None:
-        dtype = np.result_type(m, np.float64)
-    else:
-        y = np.asarray(y)
-        if y.ndim > 2:
-            raise ValueError("y has more than 2 dimensions")
-        dtype = np.result_type(m, y, np.float64)
+    if dtype is None:
+        dtype = np.float64
+        if y is None:
+            dtype = np.result_type(m, dtype)
+        else:
+            y = np.asarray(y)
+            if y.ndim > 2:
+                raise ValueError("y has more than 2 dimensions")
+            dtype = np.result_type(m, y, dtype)
path: numpy/lib/function_base.py
text: @user1:
You need to leave the `ValueError` outside the `dtype` if
```suggestion
    if y is not None:
        y = np.asarray(y)
        if y.ndim > 2:
            raise ValueError("y has more than 2 dimensions")

    if dtype is None:
        if y is None:
            dtype = np.result_type(m, np.float64)
        else:
            dtype = np.result_type(m, y, np.float64)
```

diff_hunk: @@ -2023,6 +2023,15 @@ def test_extreme(self):
         assert_array_almost_equal(c, np.array([[1., -1.], [-1., 1.]]))
         assert_(np.all(np.abs(c) <= 1.0))
 
+    @pytest.mark.parametrize("test_type", [getattr(np, dtype, None) for dtype \
+    in ['float32', 'float64', 'float96', 'float128']])
+    def test_cov_dtype(self, test_type):
+        if test_type is None:
+            pytest.skip("Data type not available")
path: numpy/lib/tests/test_function_base.py
text: @user2:
Rather than skipping, just use the names which always exist:
```suggestion
    @user1.mark.parametrize("test_type", [np.half, np.single, np.double, np.longdouble])
    def test_cov_dtype(self, test_type):
```

@author:
Nice, I wasnt aware of those. Thats much better, thanks!

@user2:
The docs aren't clear about that yet, so it's understandable you missed them - I'm updating them in #17331. The link at the top of that PR will take you to the new version.

diff_hunk: @@ -2268,13 +2268,13 @@ def _vectorize_call_with_signature(self, func, args):
 
 
 def _cov_dispatcher(m, y=None, rowvar=None, bias=None, ddof=None,
-                    fweights=None, aweights=None):
+                    fweights=None, aweights=None, dtype=None):
     return (m, y, fweights, aweights)
 
 
 @array_function_dispatch(_cov_dispatcher)
 def cov(m, y=None, rowvar=True, bias=False, ddof=None, fweights=None,
-        aweights=None):
+        aweights=None, dtype=None):
path: numpy/lib/function_base.py
text: @user1:
Let's make this keyword-only. (other arguments aren't done that way because the syntax didn't exist when this function was written)
```suggestion
        aweights=None, *, dtype=None):
```