Repository: modin-project/modin

Issue:
df.describe(include='all') generates error
### System information
- OS: macOS 12.4
- Modin version:  0.15.2
- 3.9.12 (Anaconda)
- **Code we can use to reproduce**:

In Jupyter notebook:

```
import os
os.environ["MODIN_ENGINE"] = "ray"
import modin.pandas as pd
import numpy as np
import ray
ray.init()
df = pd.DataFrame(np.zeros((2,60)))
df['new'] = "a"
df.describe(include='all').T
```

<!--
You can obtain the Modin version with

python -c "import modin; print(modin.__version__)"
-->

### Describe the problem
Error dump instead of same output as pandas


### Source code / logs
<!-- Include any logs or source code that would be helpful to diagnose the problem. If including tracebacks, please include the full traceback. Large logs and files should be attached. Try to provide a reproducible test case that is the bare minimum necessary to generate the problem. -->

Here's the error dump:

```
---------------------------------------------------------------------------
Exception                                 Traceback (most recent call last)
Input In [4], in <cell line: 9>()
      7 df = pd.DataFrame(np.zeros((2,60)))
      8 df['new'] = "a"
----> 9 df.describe(include='all').T

File ~/anaconda3/lib/python3.9/site-packages/modin/pandas/base.py:3878, in BasePandasDataset.__getattribute__(self, item)
   3865 def __getattribute__(self, item):
   3866     """
   3867     Return item from the `BasePandasDataset`.
   3868 
   (...)
   3876     Any
   3877     """
-> 3878     attr = super().__getattribute__(item)
   3879     if item not in _DEFAULT_BEHAVIOUR and not self._query_compiler.lazy_execution:
   3880         # We default to pandas on empty DataFrames. This avoids a large amount of
   3881         # pain in underlying implementation and returns a result immediately rather
   3882         # than dealing with the edge cases that empty DataFrames have.
   3883         if callable(attr) and self.empty and hasattr(self._pandas_class, item):

File ~/anaconda3/lib/python3.9/site-packages/modin/pandas/dataframe.py:525, in DataFrame.transpose(self, copy, *args)
    520 """
    521 Transpose index and columns.
    522 """
    523 # FIXME: Judging by pandas docs `*args` serves only compatibility purpose
    524 # and does not affect the result, we shouldn't pass it to the query compiler.
--> 525 return DataFrame(query_compiler=self._query_compiler.transpose(*args))

File ~/anaconda3/lib/python3.9/site-packages/modin/logging/logger_metaclass.py:68, in logger_class_wrapper.<locals>.log_wrap(*args, **kwargs)
     66     logger.info(f"END::PANDAS-API::{class_name}.{method_name}")
     67     return result
---> 68 return method(*args, **kwargs)

File ~/anaconda3/lib/python3.9/site-packages/modin/core/storage_formats/pandas/query_compiler.py:672, in PandasQueryCompiler.transpose(self, *args, **kwargs)
    670 def transpose(self, *args, **kwargs):
    671     # Switch the index and columns and transpose the data within the blocks.
--> 672     return self.__constructor__(self._modin_frame.transpose())

File ~/anaconda3/lib/python3.9/site-packages/modin/logging/logger_metaclass.py:68, in logger_class_wrapper.<locals>.log_wrap(*args, **kwargs)
     66     logger.info(f"END::PANDAS-API::{class_name}.{method_name}")
     67     return result
---> 68 return method(*args, **kwargs)

File ~/anaconda3/lib/python3.9/site-packages/modin/core/dataframe/pandas/dataframe/dataframe.py:115, in lazy_metadata_decorator.<locals>.decorator.<locals>.run_f_on_minimally_updated_metadata(self, *args, **kwargs)
    113     elif apply_axis == "rows":
    114         obj._propagate_index_objs(axis=0)
--> 115 result = f(self, *args, **kwargs)
    116 if apply_axis is None and not transpose:
    117     result._deferred_index = self._deferred_index

File ~/anaconda3/lib/python3.9/site-packages/modin/core/dataframe/pandas/dataframe/dataframe.py:2887, in PandasDataframe.transpose(self)
   2885 else:
   2886     new_dtypes = None
-> 2887 return self.__constructor__(
   2888     new_partitions,
   2889     self.columns,
   2890     self.index,
   2891     self._column_widths,
   2892     self._row_lengths,
   2893     dtypes=new_dtypes,
   2894 )

File ~/anaconda3/lib/python3.9/site-packages/modin/logging/logger_metaclass.py:68, in logger_class_wrapper.<locals>.log_wrap(*args, **kwargs)
     66     logger.info(f"END::PANDAS-API::{class_name}.{method_name}")
     67     return result
---> 68 return method(*args, **kwargs)

File ~/anaconda3/lib/python3.9/site-packages/modin/core/dataframe/pandas/dataframe/dataframe.py:210, in PandasDataframe.__init__(self, partitions, index, columns, row_lengths, column_widths, dtypes)
    208     num_columns = sum(column_widths)
    209     if num_columns > 0:
--> 210         ErrorMessage.catch_bugs_and_request_email(
    211             num_columns != len(self._columns_cache),
    212             "Column widths: {} != {}".format(
    213                 num_columns, len(self._columns_cache)
    214             ),
    215         )
    216     ErrorMessage.catch_bugs_and_request_email(
    217         any(val < 0 for val in column_widths),
    218         "Column widths cannot be negative: {}".format(column_widths),
    219     )
    220 self._column_widths_cache = column_widths

File ~/anaconda3/lib/python3.9/site-packages/modin/error_message.py:70, in ErrorMessage.catch_bugs_and_request_email(cls, failure_condition, extra_log)
     68 if failure_condition:
     69     get_logger().info(f"Modin Error: Internal Error: {extra_log}")
---> 70     raise Exception(
     71         "Internal Error. "
     72         + "Please visit https://github.com/modin-project/modin/issues "
     73         + "to file an issue with the traceback and the command that "
     74         + "caused this error. If you can't file a GitHub issue, "
     75         + f"please email bug_reports@modin.org.\n{extra_log}"
     76     )

Exception: Internal Error. Please visit https://github.com/modin-project/modin/issues to file an issue with the traceback and the command that caused this error. If you can't file a GitHub issue, please email bug_reports@modin.org.
Column widths: 8 != 11
```


Discussion:
@rhstanton thank you for reporting this bug! Your script reproduces the error for me at Modin version 910bf2dd893f52072c2c25413bf0866825f8309d. 

The reason for the bug is that each column partition individually calculates `describe`, but in this case, partitions 0 and 1 get different `describe` properties. Partition 0 only has the zeros, all of type `float64`, so its describe result has `Index(['count', 'mean', 'std', 'min', '25%', '50%', '75%', 'max'], dtype='object')` of length 8. Partition 1 has more zeros as well as the `'new'`column of type `object`, so its describe result has `Index(['count', 'unique', 'top', 'freq', 'mean', 'std', 'min', '25%', '50%', '75%', 'max'], dtype='object')` of length 11. Partition 0's index is inconsistent with the global dataframe index, which is equal to the length-11 index for partition 1, hence the bug. Maybe we need to expand the summary rows from each partition to all the rows that are in the global index
For comparison, here's the output when the same code is run in pandas:

```
	count	unique	top	freq	mean	std	min	25%	50%	75%	max
0	2.0	NaN	NaN	NaN	0.0	0.0	0.0	0.0	0.0	0.0	0.0
1	2.0	NaN	NaN	NaN	0.0	0.0	0.0	0.0	0.0	0.0	0.0
2	2.0	NaN	NaN	NaN	0.0	0.0	0.0	0.0	0.0	0.0	0.0
3	2.0	NaN	NaN	NaN	0.0	0.0	0.0	0.0	0.0	0.0	0.0
4	2.0	NaN	NaN	NaN	0.0	0.0	0.0	0.0	0.0	0.0	0.0
...	...	...	...	...	...	...	...	...	...	...	...
56	2.0	NaN	NaN	NaN	0.0	0.0	0.0	0.0	0.0	0.0	0.0
57	2.0	NaN	NaN	NaN	0.0	0.0	0.0	0.0	0.0	0.0	0.0
58	2.0	NaN	NaN	NaN	0.0	0.0	0.0	0.0	0.0	0.0	0.0
59	2.0	NaN	NaN	NaN	0.0	0.0	0.0	0.0	0.0	0.0	0.0
new	2	1	a	2	NaN	NaN	NaN	NaN	NaN	NaN	NaN
```

Resolving patch:
diff --git a/modin/core/storage_formats/pandas/query_compiler.py b/modin/core/storage_formats/pandas/query_compiler.py
--- a/modin/core/storage_formats/pandas/query_compiler.py
+++ b/modin/core/storage_formats/pandas/query_compiler.py
@@ -1577,7 +1577,14 @@ def describe(self, **kwargs):
 
         def describe_builder(df, internal_indices=[]):
             """Apply `describe` function to the subset of columns in a single partition."""
-            return df.iloc[:, internal_indices].describe(**kwargs)
+            # The index of the resulting dataframe is the same amongst all partitions
+            # when dealing with the same data type. However, if we work with columns
+            # that contain strings, we can get extra values in our result index such as
+            # 'unique', 'top', and 'freq'. Since we call describe() on each partition,
+            # we can have cases where certain partitions do not contain any of the
+            # object string data leading to an index mismatch between partitions.
+            # Thus, we must reindex each partition with the global new_index.
+            return df.iloc[:, internal_indices].describe(**kwargs).reindex(new_index)
 
         return self.__constructor__(
             self._modin_frame.apply_full_axis_select_indices(


Test patch:
diff --git a/modin/pandas/test/dataframe/test_reduce.py b/modin/pandas/test/dataframe/test_reduce.py
--- a/modin/pandas/test/dataframe/test_reduce.py
+++ b/modin/pandas/test/dataframe/test_reduce.py
@@ -181,6 +181,16 @@ def test_2195(datetime_is_numeric, has_numeric_column):
     )
 
 
+# Issue: https://github.com/modin-project/modin/issues/4641
+def test_describe_column_partition_has_different_index():
+    pandas_df = pandas.DataFrame(test_data["int_data"])
+    # We add a string column to test the case where partitions with mixed data
+    # types have different 'describe' rows, which causes an index mismatch.
+    pandas_df["string_column"] = "abc"
+    modin_df = pd.DataFrame(pandas_df)
+    eval_general(modin_df, pandas_df, lambda df: df.describe(include="all"))
+
+
 @pytest.mark.parametrize(
     "exclude,include",
     [