Repository: modin-project/modin

Issue:
Implement `groupby.skew()` via GroupbyReduce pattern
**Is your feature request related to a problem? Please describe.**
A clear and concise description of what the problem is. What kind of performance improvements would you like to see with this new API?



Discussion:

Resolving patch:
diff --git a/modin/core/dataframe/algebra/groupby.py b/modin/core/dataframe/algebra/groupby.py
--- a/modin/core/dataframe/algebra/groupby.py
+++ b/modin/core/dataframe/algebra/groupby.py
@@ -80,6 +80,7 @@ def map(
         agg_kwargs,
         other=None,
         by=None,
+        drop=False,
     ):
         """
         Execute Map phase of GroupByReduce.
@@ -108,6 +109,8 @@ def map(
         by : level index name or list of such labels, optional
             Index levels, that is used to determine groups.
             If not specified, `other` parameter is used.
+        drop : bool, default: False
+            Indicates whether or not by-data came from the `self` frame.
 
         Returns
         -------
@@ -124,10 +127,12 @@ def map(
         # present in the func-dict:
         apply_func = cls.try_filter_dict(map_func, df)
         if other is not None:
-            # Other is a broadcasted partition that represents 'by' columns
-            # Concatenate it with 'df' to group on its columns names
-            other = other.squeeze(axis=axis ^ 1)
-            if isinstance(other, pandas.DataFrame):
+            # Other is a broadcasted partition that represents 'by' data to group on.
+            # If 'drop' then the 'by' data came from the 'self' frame, thus
+            # inserting missed columns to the partition to group on them.
+            if drop or isinstance(
+                other := other.squeeze(axis=axis ^ 1), pandas.DataFrame
+            ):
                 df = pandas.concat(
                     [df] + [other[[o for o in other if o not in df]]],
                     axis=1,
@@ -439,6 +444,7 @@ def wrapper(df, other=None):
                     map_func=map_func,
                     agg_args=agg_args,
                     agg_kwargs=agg_kwargs,
+                    drop=drop,
                     **kwargs,
                 )
 
diff --git a/modin/core/storage_formats/pandas/query_compiler.py b/modin/core/storage_formats/pandas/query_compiler.py
--- a/modin/core/storage_formats/pandas/query_compiler.py
+++ b/modin/core/storage_formats/pandas/query_compiler.py
@@ -2707,6 +2707,67 @@ def groupby_size(
             )
         return result
 
+    def groupby_skew(self, by, axis, groupby_kwargs, agg_args, agg_kwargs, drop=False):
+        def map_skew(dfgb, *args, **kwargs):
+            df = dfgb.obj
+            by_cols = dfgb.exclusions
+            cols_to_agg = df.columns.difference(by_cols)
+
+            df_pow2 = pandas.concat([df[by_cols], df[cols_to_agg] ** 2], axis=1)
+            df_pow3 = pandas.concat([df[by_cols], df[cols_to_agg] ** 3], axis=1)
+
+            return pandas.concat(
+                [
+                    dfgb.count(*args, **kwargs),
+                    dfgb.sum(*args, **kwargs),
+                    df_pow2.groupby(dfgb.keys, **groupby_kwargs).sum(*args, **kwargs),
+                    df_pow3.groupby(dfgb.keys, **groupby_kwargs).sum(*args, **kwargs),
+                ],
+                copy=False,
+                axis=1,
+            )
+
+        def reduce_skew(dfgb, *args, **kwargs):
+            df = dfgb.sum(*args, **kwargs)
+            chunk_size = df.shape[1] // 4
+
+            count = df.iloc[:, :chunk_size]
+            # s = sum(x)
+            s = df.iloc[:, chunk_size : chunk_size * 2]
+            # s2 = sum(x^2)
+            s2 = df.iloc[:, chunk_size * 2 : chunk_size * 3]
+            # s3 = sum(x^3)
+            s3 = df.iloc[:, chunk_size * 3 : chunk_size * 4]
+
+            # mean = sum(x) / count
+            m = s / count
+
+            # m2 = sum( (x - m)^ 2) = sum(x^2 - 2*x*m + m^2)
+            m2 = s2 - 2 * m * s + count * (m**2)
+
+            # m3 = sum( (x - m)^ 3) = sum(x^3 - 3*x^2*m + 3*x*m^2 - m^3)
+            m3 = s3 - 3 * m * s2 + 3 * s * (m**2) - count * (m**3)
+
+            # The equation for the 'skew' was taken directly from pandas:
+            # https://github.com/pandas-dev/pandas/blob/8dab54d6573f7186ff0c3b6364d5e4dd635ff3e7/pandas/core/nanops.py#L1226
+            skew_res = (count * (count - 1) ** 0.5 / (count - 2)) * (m3 / m2**1.5)
+            return skew_res
+
+        result = GroupByReduce.register(
+            map_skew,
+            reduce_skew,
+            default_to_pandas_func=lambda dfgb, **kwargs: dfgb.skew(**kwargs),
+        )(
+            query_compiler=self,
+            by=by,
+            axis=axis,
+            groupby_kwargs=groupby_kwargs,
+            agg_args=agg_args,
+            agg_kwargs=agg_kwargs,
+            drop=drop,
+        )
+        return result
+
     def _groupby_dict_reduce(
         self,
         by,
diff --git a/modin/pandas/groupby.py b/modin/pandas/groupby.py
--- a/modin/pandas/groupby.py
+++ b/modin/pandas/groupby.py
@@ -120,11 +120,38 @@ def ngroups(self):
         return len(self)
 
     def skew(self, *args, **kwargs):
-        return self._wrap_aggregation(
-            type(self._query_compiler).groupby_skew,
+        # The 'skew' aggregation is less tolerant to non-numeric columns than others
+        # (i.e. it doesn't allow numeric categoricals), thus dropping non-numeric
+        # columns here since `._wrap_aggregation(numeric_only=True, ...)` is not enough
+        if self.ndim == 2:
+            by_cols = self._internal_by
+            mask_cols = [
+                col
+                for col, dtype in self._df.dtypes.items()
+                if is_numeric_dtype(dtype) or col in by_cols
+            ]
+            if not self._df.columns.equals(mask_cols):
+                masked_df = self._df[mask_cols]
+                masked_obj = type(self)(
+                    df=masked_df,
+                    by=self._by,
+                    axis=self._axis,
+                    idx_name=self._idx_name,
+                    drop=self._drop,
+                    squeeze=self._squeeze,
+                    **self._kwargs,
+                )
+            else:
+                masked_obj = self
+        else:
+            masked_obj = self
+
+        return masked_obj._wrap_aggregation(
+            type(masked_obj._query_compiler).groupby_skew,
             agg_args=args,
             agg_kwargs=kwargs,
-            numeric_only=True,
+            # Don't want to try to drop non-numeric columns for the second time
+            numeric_only=False,
         )
 
     def ffill(self, limit=None):


Test patch: