Repository: modin-project/modin

Issue:
corr function implementation



Discussion:

Resolving patch:
diff --git a/modin/backends/base/query_compiler.py b/modin/backends/base/query_compiler.py
--- a/modin/backends/base/query_compiler.py
+++ b/modin/backends/base/query_compiler.py
@@ -270,6 +270,12 @@ def mod(self, other, **kwargs):
     def mul(self, other, **kwargs):
         return BinaryDefault.register(pandas.DataFrame.mul)(self, other=other, **kwargs)
 
+    def corr(self, **kwargs):
+        return DataFrameDefault.register(pandas.DataFrame.corr)(self, **kwargs)
+
+    def cov(self, **kwargs):
+        return DataFrameDefault.register(pandas.DataFrame.cov)(self, **kwargs)
+
     def dot(self, other, **kwargs):
         if kwargs.get("squeeze_self", False):
             applyier = pandas.Series.dot
diff --git a/modin/backends/pandas/query_compiler.py b/modin/backends/pandas/query_compiler.py
--- a/modin/backends/pandas/query_compiler.py
+++ b/modin/backends/pandas/query_compiler.py
@@ -1226,6 +1226,9 @@ def stack(self, level, dropna):
     invert = MapFunction.register(pandas.DataFrame.__invert__)
     isin = MapFunction.register(pandas.DataFrame.isin, dtypes=np.bool)
     isna = MapFunction.register(pandas.DataFrame.isna, dtypes=np.bool)
+    _isfinite = MapFunction.register(
+        lambda df, *args, **kwargs: pandas.DataFrame(np.isfinite(df))
+    )
     negative = MapFunction.register(pandas.DataFrame.__neg__)
     notna = MapFunction.register(pandas.DataFrame.notna, dtypes=np.bool)
     round = MapFunction.register(pandas.DataFrame.round)
@@ -1589,6 +1592,91 @@ def clip(self, lower, upper, **kwargs):
             new_modin_frame = self._modin_frame._map(lambda df: df.clip(**kwargs))
         return self.__constructor__(new_modin_frame)
 
+    def corr(self, method="pearson", min_periods=1):
+        if method == "pearson":
+            numeric_self = self.drop(
+                columns=[
+                    i for i in self.dtypes.index if not is_numeric_dtype(self.dtypes[i])
+                ]
+            )
+            return numeric_self._nancorr(min_periods=min_periods)
+        else:
+            return super().corr(method=method, min_periods=min_periods)
+
+    def cov(self, min_periods=None):
+        return self._nancorr(min_periods=min_periods, cov=True)
+
+    def _nancorr(self, min_periods=1, cov=False):
+        """
+        Compute either pairwise covariance or pairwise correlation of columns,
+        considering NA/null values the same like pandas does.
+
+        Parameters
+        ----------
+        min_periods : int, default 1
+            Minimum number of observations required per pair of columns
+            to have a valid result.
+        cov : boolean, default False
+            Either covariance or correlation should be computed.
+
+        Returns
+        -------
+        PandasQueryCompiler
+            The covariance or correlation matrix of the series of the DataFrame.
+        """
+        other = self.to_numpy()
+        other_mask = self._isfinite().to_numpy()
+        n_cols = other.shape[1]
+
+        if min_periods is None:
+            min_periods = 1
+
+        def map_func(df):
+            df = df.to_numpy()
+            n_rows = df.shape[0]
+            df_mask = np.isfinite(df)
+
+            result = np.empty((n_rows, n_cols), dtype=np.float64)
+
+            for i in range(n_rows):
+                df_ith_row = df[i]
+                df_ith_mask = df_mask[i]
+
+                for j in range(n_cols):
+                    other_jth_col = other[:, j]
+
+                    valid = df_ith_mask & other_mask[:, j]
+
+                    vx = df_ith_row[valid]
+                    vy = other_jth_col[valid]
+
+                    nobs = len(vx)
+
+                    if nobs < min_periods:
+                        result[i, j] = np.nan
+                    else:
+                        vx = vx - vx.mean()
+                        vy = vy - vy.mean()
+                        sumxy = (vx * vy).sum()
+                        sumxx = (vx * vx).sum()
+                        sumyy = (vy * vy).sum()
+
+                        denom = (nobs - 1.0) if cov else np.sqrt(sumxx * sumyy)
+                        if denom != 0:
+                            result[i, j] = sumxy / denom
+                        else:
+                            result[i, j] = np.nan
+
+            return pandas.DataFrame(result)
+
+        columns = self.columns
+        index = columns.copy()
+        transponed_self = self.transpose()
+        new_modin_frame = transponed_self._modin_frame._apply_full_axis(
+            1, map_func, new_index=index, new_columns=columns
+        )
+        return transponed_self.__constructor__(new_modin_frame)
+
     def dot(self, other, squeeze_self=None, squeeze_other=None):
         """
         Computes the matrix multiplication of self and other.
diff --git a/modin/pandas/dataframe.py b/modin/pandas/dataframe.py
--- a/modin/pandas/dataframe.py
+++ b/modin/pandas/dataframe.py
@@ -707,8 +707,43 @@ def compare(
         )
 
     def corr(self, method="pearson", min_periods=1):
-        return self._default_to_pandas(
-            pandas.DataFrame.corr, method=method, min_periods=min_periods
+        """
+        Compute pairwise correlation of columns, excluding NA/null values.
+
+        Parameters
+        ----------
+        method : {'pearson', 'kendall', 'spearman'} or callable
+            Method of correlation:
+
+            * pearson : standard correlation coefficient
+            * kendall : Kendall Tau correlation coefficient
+            * spearman : Spearman rank correlation
+            * callable: callable with input two 1d ndarrays
+                and returning a float. Note that the returned matrix from corr
+                will have 1 along the diagonals and will be symmetric
+                regardless of the callable's behavior.
+
+        min_periods : int, optional
+            Minimum number of observations required per pair of columns
+            to have a valid result. Currently only available for Pearson
+            and Spearman correlation.
+
+        Returns
+        -------
+        DataFrame
+            Correlation matrix.
+
+        Notes
+        -----
+        Correlation floating point precision may slightly differ from pandas.
+
+        For now pearson method is available only. For other methods defaults to pandas.
+        """
+        return self.__constructor__(
+            query_compiler=self._query_compiler.corr(
+                method=method,
+                min_periods=min_periods,
+            )
         )
 
     def corrwith(self, other, axis=0, drop=False, method="pearson"):
@@ -763,9 +798,7 @@ def cov(self, min_periods=None, ddof: Optional[int] = 1):
             ]
         )
 
-        cols = numeric_df.columns
-        idx = cols.copy()
-        denom = None
+        is_notna = True
 
         if all(numeric_df.notna().all()):
             if min_periods is not None and min_periods > len(numeric_df):
@@ -773,20 +806,25 @@ def cov(self, min_periods=None, ddof: Optional[int] = 1):
                 result.fill(np.nan)
                 return numeric_df.__constructor__(result)
             else:
+                cols = numeric_df.columns
+                idx = cols.copy()
                 numeric_df = numeric_df.astype(dtype="float64")
                 denom = 1.0 / (len(numeric_df) - ddof)
                 means = numeric_df.mean(axis=0)
                 result = numeric_df - means
                 result = result.T._query_compiler.conj().dot(result._query_compiler)
         else:
-            result = numeric_df._query_compiler.default_to_pandas(
-                pandas.DataFrame.cov, min_periods=min_periods, ddof=ddof
-            )
+            result = numeric_df._query_compiler.cov(min_periods=min_periods)
+            is_notna = False
 
-        result = numeric_df.__constructor__(
-            query_compiler=result, index=idx, columns=cols
-        )
-        return result if denom is None else result * denom
+        if is_notna:
+            result = numeric_df.__constructor__(
+                query_compiler=result, index=idx, columns=cols
+            )
+            result *= denom
+        else:
+            result = numeric_df.__constructor__(query_compiler=result)
+        return result
 
     def dot(self, other):
         """


Test patch:
diff --git a/modin/pandas/test/dataframe/test_default.py b/modin/pandas/test/dataframe/test_default.py
--- a/modin/pandas/test/dataframe/test_default.py
+++ b/modin/pandas/test/dataframe/test_default.py
@@ -36,6 +36,7 @@
     test_data_resample,
     test_data,
     test_data_diff_dtype,
+    modin_df_almost_equals_pandas,
 )
 
 pd.DEFAULT_NPARTITIONS = 4
@@ -48,7 +49,6 @@
     "op, make_args",
     [
         ("align", lambda df: {"other": df}),
-        ("corr", None),
         ("expanding", None),
         ("corrwith", lambda df: {"other": df}),
         ("explode", lambda df: {"column": df.columns[0]}),
@@ -214,11 +214,35 @@ def test_combine_first():
     df_equals(modin_df1.combine_first(modin_df2), pandas_df1.combine_first(pandas_df2))
 
 
-def test_cov():
-    data = test_data_values[0]
-    modin_result = pd.DataFrame(data).cov()
-    pandas_result = pandas.DataFrame(data).cov()
-    df_equals(modin_result, pandas_result)
+@pytest.mark.parametrize("min_periods", [1, 3, 5])
+def test_corr(min_periods):
+    eval_general(
+        *create_test_dfs(test_data["int_data"]),
+        lambda df: df.corr(min_periods=min_periods),
+    )
+    # Modin result may slightly differ from pandas result
+    # due to floating pointing arithmetic.
+    eval_general(
+        *create_test_dfs(test_data["float_nan_data"]),
+        lambda df: df.corr(min_periods=min_periods),
+        comparator=modin_df_almost_equals_pandas,
+    )
+
+
+@pytest.mark.parametrize("min_periods", [1, 3, 5])
+@pytest.mark.parametrize("ddof", [1, 2, 4])
+def test_cov(min_periods, ddof):
+    eval_general(
+        *create_test_dfs(test_data["int_data"]),
+        lambda df: df.cov(min_periods=min_periods, ddof=ddof),
+    )
+    # Modin result may slightly differ from pandas result
+    # due to floating pointing arithmetic.
+    eval_general(
+        *create_test_dfs(test_data["float_nan_data"]),
+        lambda df: df.cov(min_periods=min_periods),
+        comparator=modin_df_almost_equals_pandas,
+    )
 
 
 @pytest.mark.parametrize("data", test_data_values, ids=test_data_keys)
diff --git a/modin/pandas/test/test_groupby.py b/modin/pandas/test/test_groupby.py
--- a/modin/pandas/test/test_groupby.py
+++ b/modin/pandas/test/test_groupby.py
@@ -15,41 +15,20 @@
 import pandas
 import numpy as np
 import modin.pandas as pd
-from modin.utils import try_cast_to_pandas, to_pandas, get_current_backend
+from modin.utils import try_cast_to_pandas, get_current_backend
 from modin.pandas.utils import from_pandas
 from .utils import (
     df_equals,
     check_df_columns_have_nans,
     create_test_dfs,
     eval_general,
-    df_categories_equals,
     test_data_values,
+    modin_df_almost_equals_pandas,
 )
 
 pd.DEFAULT_NPARTITIONS = 4
 
 
-def modin_df_almost_equals_pandas(modin_df, pandas_df):
-    df_categories_equals(modin_df._to_pandas(), pandas_df)
-
-    modin_df = to_pandas(modin_df)
-
-    if hasattr(modin_df, "select_dtypes"):
-        modin_df = modin_df.select_dtypes(exclude=["category"])
-    if hasattr(pandas_df, "select_dtypes"):
-        pandas_df = pandas_df.select_dtypes(exclude=["category"])
-
-    difference = modin_df - pandas_df
-    diff_max = difference.max()
-    if isinstance(diff_max, pandas.Series):
-        diff_max = diff_max.max()
-    assert (
-        modin_df.equals(pandas_df)
-        or diff_max < 0.0001
-        or (all(modin_df.isna().all()) and all(pandas_df.isna().all()))
-    )
-
-
 def modin_groupby_equals_pandas(modin_groupby, pandas_groupby):
     for g1, g2 in zip(modin_groupby, pandas_groupby):
         assert g1[0] == g2[0]
diff --git a/modin/pandas/test/utils.py b/modin/pandas/test/utils.py
--- a/modin/pandas/test/utils.py
+++ b/modin/pandas/test/utils.py
@@ -548,6 +548,27 @@ def df_equals(df1, df2):
             np.testing.assert_almost_equal(df1, df2)
 
 
+def modin_df_almost_equals_pandas(modin_df, pandas_df):
+    df_categories_equals(modin_df._to_pandas(), pandas_df)
+
+    modin_df = to_pandas(modin_df)
+
+    if hasattr(modin_df, "select_dtypes"):
+        modin_df = modin_df.select_dtypes(exclude=["category"])
+    if hasattr(pandas_df, "select_dtypes"):
+        pandas_df = pandas_df.select_dtypes(exclude=["category"])
+
+    difference = modin_df - pandas_df
+    diff_max = difference.max()
+    if isinstance(diff_max, pandas.Series):
+        diff_max = diff_max.max()
+    assert (
+        modin_df.equals(pandas_df)
+        or diff_max < 0.0001
+        or (all(modin_df.isna().all()) and all(pandas_df.isna().all()))
+    )
+
+
 def df_is_empty(df):
     """Tests if df is empty.