Repository: modin-project/modin

Issue:
REFACTOR: Simplify describe() call
- https://github.com/ponder-org/modin-public/commit/5e02f8f5669266f3c43ef8b55d7d1b7f8fac3189


Discussion:

Resolving patch:
diff --git a/modin/core/storage_formats/base/query_compiler.py b/modin/core/storage_formats/base/query_compiler.py
--- a/modin/core/storage_formats/base/query_compiler.py
+++ b/modin/core/storage_formats/base/query_compiler.py
@@ -1879,25 +1879,19 @@ def var(self, **kwargs):  # noqa: PR02
 
     # END Abstract column/row partitions reduce operations
 
-    # Abstract column/row partitions reduce operations over select indices
-    #
-    # These operations result in a reduced dimensionality of data.
-    # Currently, this means a Pandas Series will be returned, but in the future
-    # we will implement a Distributed Series, and this will be returned
-    # instead.
     @doc_utils.add_refer_to("DataFrame.describe")
-    def describe(self, **kwargs):  # noqa: PR02
+    def describe(
+        self,
+        percentiles: np.ndarray,
+        datetime_is_numeric: bool,
+    ):
         """
         Generate descriptive statistics.
 
         Parameters
         ----------
         percentiles : list-like
-        include : "all" or list of dtypes, optional
-        exclude : list of dtypes, optional
         datetime_is_numeric : bool
-        **kwargs : dict
-            Serves the compatibility purpose. Does not affect the result.
 
         Returns
         -------
@@ -1905,9 +1899,12 @@ def describe(self, **kwargs):  # noqa: PR02
             QueryCompiler object containing the descriptive statistics
             of the underlying data.
         """
-        return DataFrameDefault.register(pandas.DataFrame.describe)(self, **kwargs)
-
-    # END Abstract column/row partitions reduce operations over select indices
+        return DataFrameDefault.register(pandas.DataFrame.describe)(
+            self,
+            percentiles=percentiles,
+            datetime_is_numeric=datetime_is_numeric,
+            include="all",
+        )
 
     # Map across rows/columns
     # These operations require some global knowledge of the full column/row
diff --git a/modin/core/storage_formats/pandas/query_compiler.py b/modin/core/storage_formats/pandas/query_compiler.py
--- a/modin/core/storage_formats/pandas/query_compiler.py
+++ b/modin/core/storage_formats/pandas/query_compiler.py
@@ -1756,24 +1756,24 @@ def last_valid_index_builder(df):
 
     # END Column/Row partitions reduce operations
 
-    # Column/Row partitions reduce operations over select indices
-    #
-    # These operations result in a reduced dimensionality of data.
-    # This will return a new QueryCompiler object which the front end will handle.
-
-    def describe(self, **kwargs):
+    def describe(
+        self,
+        percentiles: np.ndarray,
+        datetime_is_numeric: bool,
+    ):
         # Use pandas to calculate the correct columns
         empty_df = (
             pandas.DataFrame(columns=self.columns)
             .astype(self.dtypes)
-            .describe(**kwargs)
+            .describe(
+                percentiles, datetime_is_numeric=datetime_is_numeric, include="all"
+            )
         )
         new_index = empty_df.index
 
         # Note: `describe` convert timestamp type to object type
         # which results in the loss of two values in index: `first` and `last`
         # for empty DataFrame.
-        datetime_is_numeric = kwargs.get("datetime_is_numeric") or False
         if not any(map(is_numeric_dtype, empty_df.dtypes)) and not datetime_is_numeric:
             for col_name in empty_df.dtypes.index:
                 # if previosly type of `col_name` was datetime or timedelta
@@ -1792,7 +1792,15 @@ def describe_builder(df, internal_indices=[]):  # pragma: no cover
             # we can have cases where certain partitions do not contain any of the
             # object string data leading to an index mismatch between partitions.
             # Thus, we must reindex each partition with the global new_index.
-            return df.iloc[:, internal_indices].describe(**kwargs).reindex(new_index)
+            return (
+                df.iloc[:, internal_indices]
+                .describe(
+                    percentiles=percentiles,
+                    datetime_is_numeric=datetime_is_numeric,
+                    include="all",
+                )
+                .reindex(new_index)
+            )
 
         return self.__constructor__(
             self._modin_frame.apply_full_axis_select_indices(
diff --git a/modin/pandas/base.py b/modin/pandas/base.py
--- a/modin/pandas/base.py
+++ b/modin/pandas/base.py
@@ -17,6 +17,7 @@
 import pandas
 from pandas.compat import numpy as numpy_compat
 from pandas.core.common import count_not_none, pipe
+from pandas.core.describe import refine_percentiles
 from pandas.core.dtypes.common import (
     is_list_like,
     is_dict_like,
@@ -24,7 +25,6 @@
     is_datetime_or_timedelta_dtype,
     is_dtype_equal,
     is_object_dtype,
-    pandas_dtype,
     is_integer,
 )
 from pandas.core.indexes.api import ensure_index
@@ -50,6 +50,7 @@
     TimedeltaConvertibleTypes,
     TimestampConvertibleTypes,
     RandomState,
+    npt,
 )
 import pickle as pkl
 import re
@@ -1130,6 +1131,8 @@ def count(self, axis=0, level=None, numeric_only=False):  # noqa: PR01, RT01, D2
         Count non-NA cells for `BasePandasDataset`.
         """
         axis = self._get_axis_number(axis)
+        # select_dtypes is only implemented on DataFrames, but the numeric_only
+        # flag will always be set to false by the Series frontend
         frame = self.select_dtypes([np.number, np.bool_]) if numeric_only else self
 
         if level is not None:
@@ -1208,54 +1211,35 @@ def describe(
         """
         Generate descriptive statistics.
         """
-        if include is not None and (isinstance(include, np.dtype) or include != "all"):
-            if not is_list_like(include):
-                include = [include]
-            include = [pandas_dtype(i) if i != np.number else i for i in include]
-            if not any(
-                (isinstance(inc, np.dtype) and inc == d)
-                or (
-                    not isinstance(inc, np.dtype)
-                    and inc.__subclasscheck__(getattr(np, d.__str__()))
-                )
-                for d in self._get_dtypes()
-                for inc in include
-            ):
-                # This is the error that pandas throws.
-                raise ValueError("No objects to concatenate")
-        if exclude is not None:
-            if not is_list_like(exclude):
-                exclude = [exclude]
-            exclude = [pandas_dtype(e) if e != np.number else e for e in exclude]
-            if all(
-                (isinstance(exc, np.dtype) and exc == d)
-                or (
-                    not isinstance(exc, np.dtype)
-                    and exc.__subclasscheck__(getattr(np, d.__str__()))
+        # copied from pandas.core.describe.describe_ndframe
+        percentiles = refine_percentiles(percentiles)
+        data = self
+        if self._is_dataframe:
+            # include/exclude arguments are ignored for Series
+            if (include is None) and (exclude is None):
+                # when some numerics are found, keep only numerics
+                default_include: list[npt.DTypeLike] = [np.number]
+                if datetime_is_numeric:
+                    default_include.append("datetime")
+                data = self.select_dtypes(include=default_include)
+                if len(data.columns) == 0:
+                    data = self
+            elif include == "all":
+                if exclude is not None:
+                    msg = "exclude must be None when include is 'all'"
+                    raise ValueError(msg)
+                data = self
+            else:
+                data = self.select_dtypes(
+                    include=include,
+                    exclude=exclude,
                 )
-                for d in self._get_dtypes()
-                for exc in exclude
-            ):
-                # This is the error that pandas throws.
-                raise ValueError("No objects to concatenate")
-        if percentiles is not None:
-            # explicit conversion of `percentiles` to list
-            percentiles = list(percentiles)
-
-            # get them all to be in [0, 1]
-            validate_percentile(percentiles)
-
-            # median should always be included
-            if 0.5 not in percentiles:
-                percentiles.append(0.5)
-            percentiles = np.asarray(percentiles)
-        else:
-            percentiles = np.array([0.25, 0.5, 0.75])
+        if data.empty:
+            # Match pandas error from concatenting empty list of series descriptions.
+            raise ValueError("No objects to concatenate")
         return self.__constructor__(
-            query_compiler=self._query_compiler.describe(
+            query_compiler=data._query_compiler.describe(
                 percentiles=percentiles,
-                include=include,
-                exclude=exclude,
                 datetime_is_numeric=datetime_is_numeric,
             )
         )


Test patch: