Repository: modin-project/modin

Issue:
describe ignores some include parameters when they are added to the include list

### Describe the problem
<!-- Describe the problem clearly here. -->
Modin DataFrame `describe` does not properly include the requested dtypes. We always ignore the default. This creates problems when users pass in `df.describe(include=["O"])`. We just need to selectively ignore certain dtypes.

```
RayTaskError                              Traceback (most recent call last)
~/Downloads/kaggle5/kaggle5.py in <module>
     51 
     52 
---> 53 train_df.describe(include=['O'])
     54 
     55 # In[9]:

~/software_builds/modin/modin/pandas/dataframe.py in describe(self, percentiles, include, exclude)
   1093         return DataFrame(
   1094             query_compiler=self._query_compiler.describe(
-> 1095                 percentiles=percentiles, include=include, exclude=exclude
   1096             )
   1097         )

~/software_builds/modin/modin/data_management/query_compiler/pandas_query_compiler.py in describe(self, **kwargs)
   1593             func, 0, new_columns, False
   1594         )
-> 1595         new_index = self.compute_index(0, new_data, False)
   1596         if numeric:
   1597             new_dtypes = pandas.Series(

~/software_builds/modin/modin/data_management/query_compiler/pandas_query_compiler.py in compute_index(self, axis, data_object, compute_diff)
    136             axis=axis,
    137             index_func=lambda df: pandas_index_extraction(df, axis),
--> 138             old_blocks=old_blocks,
    139         )
    140         return index_obj[new_indices] if compute_diff else new_indices

~/software_builds/modin/modin/engines/base/block_partitions.py in get_indices(self, axis, index_func, old_blocks)
    514             new_indices = (
    515                 [idx.apply(func).get() for idx in self._partitions_cache.T[0]]
--> 516                 if len(self._partitions_cache.T)
    517                 else []
    518             )

~/software_builds/modin/modin/engines/base/block_partitions.py in <listcomp>(.0)
    513             # DO NOT CHANGE TO self.partitions under any circumstance.
    514             new_indices = (
--> 515                 [idx.apply(func).get() for idx in self._partitions_cache.T[0]]
    516                 if len(self._partitions_cache.T)
    517                 else []

~/software_builds/modin/modin/engines/ray/pandas_on_ray/remote_partition.py in get(self)
     26             return self.apply(lambda x: x).get()
     27 
---> 28         return ray.get(self.oid)
     29 
     30     def apply(self, func, **kwargs):

/Library/Frameworks/Python.framework/Versions/3.6/lib/python3.6/site-packages/ray/worker.py in get(object_ids, worker)
   2384                 # here.
   2385                 last_task_error_raise_time = time.time()
-> 2386                 raise value
   2387             return value
   2388 

RayTaskError: ray_worker:modin.engines.ray.pandas_on_ray.remote_partition.deploy_ray_func() (pid=17340, host=iMac.local)
ray.worker.RayTaskError: ray_worker:modin.engines.ray.pandas_on_ray.axis_partition.deploy_ray_axis_func() (pid=17341, host=iMac.local)
  File "/Users/DevinPetersohn/software_builds/modin/modin/engines/ray/pandas_on_ray/axis_partition.py", line 127, in deploy_ray_axis_func
    result = func(dataframe, **kwargs)
  File "/Users/DevinPetersohn/software_builds/modin/modin/data_management/query_compiler/pandas_query_compiler.py", line 163, in helper
    return pandas_func(df, **kwargs)
  File "/Users/DevinPetersohn/software_builds/modin/modin/data_management/query_compiler/pandas_query_compiler.py", line 1588, in describe_builder
    return pandas.DataFrame.describe(df, **kwargs)
  File "/Library/Frameworks/Python.framework/Versions/3.6/lib/python3.6/site-packages/pandas/core/generic.py", line 8574, in describe
    data = self.select_dtypes(include=include, exclude=exclude)
  File "/Library/Frameworks/Python.framework/Versions/3.6/lib/python3.6/site-packages/pandas/core/frame.py", line 3064, in select_dtypes
    inc_ex=(include & exclude)))
ValueError: include and exclude overlap on frozenset({<class 'numpy.object_'>})
```

### Source code / logs

`df.describe(include=["O"])`
<!-- Include any logs or source code that would be helpful to diagnose the problem. If including tracebacks, please include the full traceback. Large logs and files should be attached. Try to provide a reproducible test case that is the bare minimum necessary to generate the problem. -->



Discussion:

Resolving patch:
diff --git a/modin/data_management/query_compiler/pandas_query_compiler.py b/modin/data_management/query_compiler/pandas_query_compiler.py
--- a/modin/data_management/query_compiler/pandas_query_compiler.py
+++ b/modin/data_management/query_compiler/pandas_query_compiler.py
@@ -1559,10 +1559,18 @@ def describe(self, **kwargs):
         if len(new_columns) != 0:
             numeric = True
             exclude = kwargs.get("exclude", None)
+            include = kwargs.get("include", None)
+            # This is done to check against the default dtypes with 'in'.
+            # We don't change `include` in kwargs, so we can just use this for the
+            # check.
+            if include is None:
+                include = []
+            default_excludes = [np.timedelta64, np.datetime64, np.object, np.bool]
+            add_to_excludes = [e for e in default_excludes if e not in include]
             if is_list_like(exclude):
-                exclude.append([np.timedelta64, np.datetime64, np.object, np.bool])
+                exclude.append(add_to_excludes)
             else:
-                exclude = [exclude, np.timedelta64, np.datetime64, np.object, np.bool]
+                exclude = add_to_excludes
             kwargs["exclude"] = exclude
         else:
             numeric = False
diff --git a/modin/pandas/dataframe.py b/modin/pandas/dataframe.py
--- a/modin/pandas/dataframe.py
+++ b/modin/pandas/dataframe.py
@@ -1080,14 +1080,14 @@ def describe(self, percentiles=None, include=None, exclude=None):
 
         Returns: Series/DataFrame of summary statistics
         """
-        if exclude is None:
-            exclude = "object"
-        elif "object" not in include:
-            exclude = (
-                ([exclude] + ["object"])
-                if isinstance(exclude, str)
-                else list(exclude) + ["object"]
-            )
+        if include is not None:
+            if not is_list_like(include):
+                include = [include]
+            include = [np.dtype(i) for i in include]
+        if exclude is not None:
+            if not is_list_like(include):
+                exclude = [exclude]
+            exclude = [np.dtype(e) for e in exclude]
         if percentiles is not None:
             pandas.DataFrame()._check_percentile(percentiles)
         return DataFrame(


Test patch: