Repository: modin-project/modin

Issue:
df.describe() Internal Error
### System information
- **OS Platform and Distribution (e.g., Linux Ubuntu 16.04)**: Arch Linux; kernel 5.8.7-arch1-1
- **Modin version** (`modin.__version__`): 0.8.0
- **Python version**: 3.8.5 (default, Sep  5 2020, 10:50:12) \n[GCC 10.2.0]
- **Code we can use to reproduce**: `df.describe()` that requires a 1.7GB dataset, but I can provide if requested.

<!--
You can obtain the Modin version with

python -c "import modin; print(modin.__version__)"
-->

### Describe the problem
<!-- Describe the problem clearly here. -->

`df.describe()`

 in a jupyter notebook, for a df like this:

```
df.info()
<class 'modin.pandas.dataframe.DataFrame'>
RangeIndex: 102634640 entries, 0 to 102634639
Data columns (total 3 columns):
 #   Column            Non-Null Count      Dtype         
---  ----------------  ------------------  -----         
 0   STREET_BLOCK      102634640 non-null  category
 1   SESSION_START_DT  102634640 non-null  datetime64[ns]
 2   SESSION_END_DT    102634640 non-null  datetime64[ns]
dtypes: datetime64[ns](2), category(1)
memory usage: 1.7 GB
```
The df was created by `chunks = pd.read_csv(..., chunksize=1000000)` and then `pd.concat(list(chunks))`.

After saving the df to feather file, restarting and loading, I get the second traceback, which is slightly different.

### Source code / logs
<!-- Include any logs or source code that would be helpful to diagnose the problem. If including tracebacks, please include the full traceback. Large logs and files should be attached. Try to provide a reproducible test case that is the bare minimum necessary to generate the problem. -->
```
df.describe()
---------------------------------------------------------------------------
Exception                                 Traceback (most recent call last)
/usr/lib/python3.8/site-packages/IPython/core/formatters.py in __call__(self, obj)
    700                 type_pprinters=self.type_printers,
    701                 deferred_pprinters=self.deferred_printers)
--> 702             printer.pretty(obj)
    703             printer.flush()
    704             return stream.getvalue()

/usr/lib/python3.8/site-packages/IPython/lib/pretty.py in pretty(self, obj)
    392                         if cls is not object \
    393                                 and callable(cls.__dict__.get('__repr__')):
--> 394                             return _repr_pprint(obj, self, cycle)
    395 
    396             return _default_pprint(obj, self, cycle)

/usr/lib/python3.8/site-packages/IPython/lib/pretty.py in _repr_pprint(obj, p, cycle)
    698     """A pprint that just redirects to the normal repr function."""
    699     # Find newlines and replace them with p.break_()
--> 700     output = repr(obj)
    701     lines = output.splitlines()
    702     with p.group():

~/.local/lib/python3.8/site-packages/modin/pandas/dataframe.py in __repr__(self)
    163 
    164             num_cols += len(self.columns) - i
--> 165         result = repr(self._build_repr_df(num_rows, num_cols))
    166         if len(self.index) > num_rows or len(self.columns) > num_cols:
    167             # The split here is so that we don't repr pandas row lengths.

~/.local/lib/python3.8/site-packages/modin/pandas/base.py in _build_repr_df(self, num_rows, num_cols)
    106         else:
    107             indexer = row_indexer
--> 108         return self.iloc[indexer]._query_compiler.to_pandas()
    109 
    110     def _update_inplace(self, new_query_compiler):

~/.local/lib/python3.8/site-packages/modin/pandas/indexing.py in __getitem__(self, key)
    377 
    378         row_lookup, col_lookup = self._compute_lookup(row_loc, col_loc)
--> 379         result = super(_iLocIndexer, self).__getitem__(row_lookup, col_lookup, ndim)
    380         if isinstance(result, Series):
    381             result._parent = self.df

~/.local/lib/python3.8/site-packages/modin/pandas/indexing.py in __getitem__(self, row_lookup, col_lookup, ndim)
    128 
    129     def __getitem__(self, row_lookup, col_lookup, ndim):
--> 130         qc_view = self.qc.view(row_lookup, col_lookup)
    131         if ndim == 2:
    132             return self.df.__constructor__(query_compiler=qc_view)

~/.local/lib/python3.8/site-packages/modin/backends/pandas/query_compiler.py in view(self, index, columns)
   2144     def view(self, index=None, columns=None):
   2145         return self.__constructor__(
-> 2146             self._modin_frame.mask(row_numeric_idx=index, col_numeric_idx=columns)
   2147         )
   2148 

~/.local/lib/python3.8/site-packages/modin/engines/base/frame/data.py in mask(self, row_indices, row_numeric_idx, col_indices, col_numeric_idx)
    464             and col_numeric_idx is None
    465         ):
--> 466             return self.copy()
    467         if row_indices is not None:
    468             row_numeric_idx = self.index.get_indexer_for(row_indices)

~/.local/lib/python3.8/site-packages/modin/engines/base/frame/data.py in copy(self)
    635             A copied version of this object.
    636         """
--> 637         return self.__constructor__(
    638             self._partitions,
    639             self.index.copy(),

~/.local/lib/python3.8/site-packages/modin/engines/base/frame/data.py in __init__(self, partitions, index, columns, row_lengths, column_widths, dtypes, validate_axes)
     63         self._columns_cache = ensure_index(columns)
     64         if row_lengths is not None and len(self.index) > 0:
---> 65             ErrorMessage.catch_bugs_and_request_email(
     66                 sum(row_lengths) != len(self._index_cache),
     67                 "Row lengths: {} != {}".format(

~/.local/lib/python3.8/site-packages/modin/error_message.py in catch_bugs_and_request_email(cls, failure_condition, extra_log)
     49     def catch_bugs_and_request_email(cls, failure_condition, extra_log=""):
     50         if failure_condition:
---> 51             raise Exception(
     52                 "Internal Error. "
     53                 "Please email bug_reports@modin.org with the traceback and command that"

Exception: Internal Error. Please email bug_reports@modin.org with the traceback and command that caused this error.
Row lengths: 6 != 4

---------------------------------------------------------------------------
Exception                                 Traceback (most recent call last)
/usr/lib/python3.8/site-packages/IPython/core/formatters.py in __call__(self, obj)
    343             method = get_real_method(obj, self.print_method)
    344             if method is not None:
--> 345                 return method()
    346             return None
    347         else:

~/.local/lib/python3.8/site-packages/modin/pandas/dataframe.py in _repr_html_(self)
    184         # We use pandas _repr_html_ to get a string of the HTML representation
    185         # of the dataframe.
--> 186         result = self._build_repr_df(num_rows, num_cols)._repr_html_()
    187         if len(self.index) > num_rows or len(self.columns) > num_cols:
    188             # We split so that we insert our correct dataframe dimensions.

~/.local/lib/python3.8/site-packages/modin/pandas/base.py in _build_repr_df(self, num_rows, num_cols)
    106         else:
    107             indexer = row_indexer
--> 108         return self.iloc[indexer]._query_compiler.to_pandas()
    109 
    110     def _update_inplace(self, new_query_compiler):

~/.local/lib/python3.8/site-packages/modin/pandas/indexing.py in __getitem__(self, key)
    377 
    378         row_lookup, col_lookup = self._compute_lookup(row_loc, col_loc)
--> 379         result = super(_iLocIndexer, self).__getitem__(row_lookup, col_lookup, ndim)
    380         if isinstance(result, Series):
    381             result._parent = self.df

~/.local/lib/python3.8/site-packages/modin/pandas/indexing.py in __getitem__(self, row_lookup, col_lookup, ndim)
    128 
    129     def __getitem__(self, row_lookup, col_lookup, ndim):
--> 130         qc_view = self.qc.view(row_lookup, col_lookup)
    131         if ndim == 2:
    132             return self.df.__constructor__(query_compiler=qc_view)

~/.local/lib/python3.8/site-packages/modin/backends/pandas/query_compiler.py in view(self, index, columns)
   2144     def view(self, index=None, columns=None):
   2145         return self.__constructor__(
-> 2146             self._modin_frame.mask(row_numeric_idx=index, col_numeric_idx=columns)
   2147         )
   2148 

~/.local/lib/python3.8/site-packages/modin/engines/base/frame/data.py in mask(self, row_indices, row_numeric_idx, col_indices, col_numeric_idx)
    464             and col_numeric_idx is None
    465         ):
--> 466             return self.copy()
    467         if row_indices is not None:
    468             row_numeric_idx = self.index.get_indexer_for(row_indices)

~/.local/lib/python3.8/site-packages/modin/engines/base/frame/data.py in copy(self)
    635             A copied version of this object.
    636         """
--> 637         return self.__constructor__(
    638             self._partitions,
    639             self.index.copy(),

~/.local/lib/python3.8/site-packages/modin/engines/base/frame/data.py in __init__(self, partitions, index, columns, row_lengths, column_widths, dtypes, validate_axes)
     63         self._columns_cache = ensure_index(columns)
     64         if row_lengths is not None and len(self.index) > 0:
---> 65             ErrorMessage.catch_bugs_and_request_email(
     66                 sum(row_lengths) != len(self._index_cache),
     67                 "Row lengths: {} != {}".format(

~/.local/lib/python3.8/site-packages/modin/error_message.py in catch_bugs_and_request_email(cls, failure_condition, extra_log)
     49     def catch_bugs_and_request_email(cls, failure_condition, extra_log=""):
     50         if failure_condition:
---> 51             raise Exception(
     52                 "Internal Error. "
     53                 "Please email bug_reports@modin.org with the traceback and command that"

Exception: Internal Error. Please email bug_reports@modin.org with the traceback and command that caused this error.
Row lengths: 6 != 4
```

After restart and load from feather file:
```
df = pd.read_feather(tmpdata)
df.describe()
---------------------------------------------------------------------------
Exception                                 Traceback (most recent call last)
/usr/lib/python3.8/site-packages/IPython/core/formatters.py in __call__(self, obj)
    700                 type_pprinters=self.type_printers,
    701                 deferred_pprinters=self.deferred_printers)
--> 702             printer.pretty(obj)
    703             printer.flush()
    704             return stream.getvalue()

/usr/lib/python3.8/site-packages/IPython/lib/pretty.py in pretty(self, obj)
    392                         if cls is not object \
    393                                 and callable(cls.__dict__.get('__repr__')):
--> 394                             return _repr_pprint(obj, self, cycle)
    395 
    396             return _default_pprint(obj, self, cycle)

/usr/lib/python3.8/site-packages/IPython/lib/pretty.py in _repr_pprint(obj, p, cycle)
    698     """A pprint that just redirects to the normal repr function."""
    699     # Find newlines and replace them with p.break_()
--> 700     output = repr(obj)
    701     lines = output.splitlines()
    702     with p.group():

~/.local/lib/python3.8/site-packages/modin/pandas/dataframe.py in __repr__(self)
    163 
    164             num_cols += len(self.columns) - i
--> 165         result = repr(self._build_repr_df(num_rows, num_cols))
    166         if len(self.index) > num_rows or len(self.columns) > num_cols:
    167             # The split here is so that we don't repr pandas row lengths.

~/.local/lib/python3.8/site-packages/modin/pandas/base.py in _build_repr_df(self, num_rows, num_cols)
    106         else:
    107             indexer = row_indexer
--> 108         return self.iloc[indexer]._query_compiler.to_pandas()
    109 
    110     def _update_inplace(self, new_query_compiler):

~/.local/lib/python3.8/site-packages/modin/backends/pandas/query_compiler.py in to_pandas(self)
    187 
    188     def to_pandas(self):
--> 189         return self._modin_frame.to_pandas()
    190 
    191     @classmethod

~/.local/lib/python3.8/site-packages/modin/engines/base/frame/data.py in to_pandas(self)
   1736                 df = pandas.DataFrame(columns=self.columns, index=self.index)
   1737         else:
-> 1738             ErrorMessage.catch_bugs_and_request_email(
   1739                 not df.index.equals(self.index) or not df.columns.equals(self.columns),
   1740                 "Internal and external indices do not match.",

~/.local/lib/python3.8/site-packages/modin/error_message.py in catch_bugs_and_request_email(cls, failure_condition, extra_log)
     49     def catch_bugs_and_request_email(cls, failure_condition, extra_log=""):
     50         if failure_condition:
---> 51             raise Exception(
     52                 "Internal Error. "
     53                 "Please email bug_reports@modin.org with the traceback and command that"

Exception: Internal Error. Please email bug_reports@modin.org with the traceback and command that caused this error.
Internal and external indices do not match.

---------------------------------------------------------------------------
Exception                                 Traceback (most recent call last)
/usr/lib/python3.8/site-packages/IPython/core/formatters.py in __call__(self, obj)
    343             method = get_real_method(obj, self.print_method)
    344             if method is not None:
--> 345                 return method()
    346             return None
    347         else:

~/.local/lib/python3.8/site-packages/modin/pandas/dataframe.py in _repr_html_(self)
    184         # We use pandas _repr_html_ to get a string of the HTML representation
    185         # of the dataframe.
--> 186         result = self._build_repr_df(num_rows, num_cols)._repr_html_()
    187         if len(self.index) > num_rows or len(self.columns) > num_cols:
    188             # We split so that we insert our correct dataframe dimensions.

~/.local/lib/python3.8/site-packages/modin/pandas/base.py in _build_repr_df(self, num_rows, num_cols)
    106         else:
    107             indexer = row_indexer
--> 108         return self.iloc[indexer]._query_compiler.to_pandas()
    109 
    110     def _update_inplace(self, new_query_compiler):

~/.local/lib/python3.8/site-packages/modin/backends/pandas/query_compiler.py in to_pandas(self)
    187 
    188     def to_pandas(self):
--> 189         return self._modin_frame.to_pandas()
    190 
    191     @classmethod

~/.local/lib/python3.8/site-packages/modin/engines/base/frame/data.py in to_pandas(self)
   1736                 df = pandas.DataFrame(columns=self.columns, index=self.index)
   1737         else:
-> 1738             ErrorMessage.catch_bugs_and_request_email(
   1739                 not df.index.equals(self.index) or not df.columns.equals(self.columns),
   1740                 "Internal and external indices do not match.",

~/.local/lib/python3.8/site-packages/modin/error_message.py in catch_bugs_and_request_email(cls, failure_condition, extra_log)
     49     def catch_bugs_and_request_email(cls, failure_condition, extra_log=""):
     50         if failure_condition:
---> 51             raise Exception(
     52                 "Internal Error. "
     53                 "Please email bug_reports@modin.org with the traceback and command that"

Exception: Internal Error. Please email bug_reports@modin.org with the traceback and command that caused this error.
Internal and external indices do not match.
```


Discussion:
Hi @TK009 thanks for the report!

Can you try to update your modin install with `pip install -U modin[ray]`? This may have been fixed in the latest release.
Please let me know if that doesn't fix the issue. Also, with Modin you don't need to read in chunks with `chunksize`, Modin works best when you let it do all the chunking for you!
Thanks for help! 

Actually `chunksize` was related to another problem, but I included it here just in case if it matters as I'm quite new to pandas/modin.

The original csv I was loading is about 12G uncompressed and I was pre-processing it to be smaller by using some `read_csv` parameters: `usecols`, `dtype` some columns to category, and `parse_dates`. Soon after modin started to process the file with all cpu cores, I run out of memory with 32G of ram.

Now that I had more time to test, the problems seems to be with `parse_dates`. When I remove that from the `read_csv` call and do it as separate calls with `to_datetime`, everything works fine without running out of memory. I don't know if this should be seen as a bug.



**Regarding the `.desrcribe` crash:** I have now installed modin v0.8.1.1, and I still get the error, but I also get some FutureWarnings now and when I change the command to `df.describe(datetime_is_numeric=True)` it doesn't crash, so it seems to be related to the datetime. So I think this issue is kind of solved now. I put the error output here in case someone needs it:
```

(pid=7557) FutureWarning: Treating datetime data as categorical rather than numeric in `.describe` is deprecated and will be removed in a future version of pandas. Specify `datetime_is_numeric=True` to silence this warning and adopt the future behavior now.
(pid=7901) FutureWarning: Treating datetime data as categorical rather than numeric in `.describe` is deprecated and will be removed in a future version of pandas. Specify `datetime_is_numeric=True` to silence this warning and adopt the future behavior now.
---------------------------------------------------------------------------
Exception                                 Traceback (most recent call last)
/usr/lib/python3.8/site-packages/IPython/core/formatters.py in __call__(self, obj)
    700                 type_pprinters=self.type_printers,
    701                 deferred_pprinters=self.deferred_printers)
--> 702             printer.pretty(obj)
    703             printer.flush()
    704             return stream.getvalue()

/usr/lib/python3.8/site-packages/IPython/lib/pretty.py in pretty(self, obj)
    392                         if cls is not object \
    393                                 and callable(cls.__dict__.get('__repr__')):
--> 394                             return _repr_pprint(obj, self, cycle)
    395 
    396             return _default_pprint(obj, self, cycle)

/usr/lib/python3.8/site-packages/IPython/lib/pretty.py in _repr_pprint(obj, p, cycle)
    698     """A pprint that just redirects to the normal repr function."""
    699     # Find newlines and replace them with p.break_()
--> 700     output = repr(obj)
    701     lines = output.splitlines()
    702     with p.group():

~/.local/lib/python3.8/site-packages/modin/pandas/dataframe.py in __repr__(self)
    168 
    169             num_cols += len(self.columns) - i
--> 170         result = repr(self._build_repr_df(num_rows, num_cols))
    171         if len(self.index) > num_rows or len(self.columns) > num_cols:
    172             # The split here is so that we don't repr pandas row lengths.

~/.local/lib/python3.8/site-packages/modin/pandas/base.py in _build_repr_df(self, num_rows, num_cols)
    117         else:
    118             indexer = row_indexer
--> 119         return self.iloc[indexer]._query_compiler.to_pandas()
    120 
    121     def _update_inplace(self, new_query_compiler):

~/.local/lib/python3.8/site-packages/modin/backends/pandas/query_compiler.py in to_pandas(self)
    193 
    194     def to_pandas(self):
--> 195         return self._modin_frame.to_pandas()
    196 
    197     @classmethod

~/.local/lib/python3.8/site-packages/modin/engines/base/frame/data.py in to_pandas(self)
   1895         else:
   1896             for axis in [0, 1]:
-> 1897                 ErrorMessage.catch_bugs_and_request_email(
   1898                     not df.axes[axis].equals(self.axes[axis]),
   1899                     f"Internal and external indices on axis {axis} do not match.",

~/.local/lib/python3.8/site-packages/modin/error_message.py in catch_bugs_and_request_email(cls, failure_condition, extra_log)
     49     def catch_bugs_and_request_email(cls, failure_condition, extra_log=""):
     50         if failure_condition:
---> 51             raise Exception(
     52                 "Internal Error. "
     53                 "Please email bug_reports@modin.org with the traceback and command that"

Exception: Internal Error. Please email bug_reports@modin.org with the traceback and command that caused this error.
Internal and external indices on axis 0 do not match.

---------------------------------------------------------------------------
Exception                                 Traceback (most recent call last)
/usr/lib/python3.8/site-packages/IPython/core/formatters.py in __call__(self, obj)
    343             method = get_real_method(obj, self.print_method)
    344             if method is not None:
--> 345                 return method()
    346             return None
    347         else:

~/.local/lib/python3.8/site-packages/modin/pandas/dataframe.py in _repr_html_(self)
    189         # We use pandas _repr_html_ to get a string of the HTML representation
    190         # of the dataframe.
--> 191         result = self._build_repr_df(num_rows, num_cols)._repr_html_()
    192         if len(self.index) > num_rows or len(self.columns) > num_cols:
    193             # We split so that we insert our correct dataframe dimensions.

~/.local/lib/python3.8/site-packages/modin/pandas/base.py in _build_repr_df(self, num_rows, num_cols)
    117         else:
    118             indexer = row_indexer
--> 119         return self.iloc[indexer]._query_compiler.to_pandas()
    120 
    121     def _update_inplace(self, new_query_compiler):

~/.local/lib/python3.8/site-packages/modin/backends/pandas/query_compiler.py in to_pandas(self)
    193 
    194     def to_pandas(self):
--> 195         return self._modin_frame.to_pandas()
    196 
    197     @classmethod

~/.local/lib/python3.8/site-packages/modin/engines/base/frame/data.py in to_pandas(self)
   1895         else:
   1896             for axis in [0, 1]:
-> 1897                 ErrorMessage.catch_bugs_and_request_email(
   1898                     not df.axes[axis].equals(self.axes[axis]),
   1899                     f"Internal and external indices on axis {axis} do not match.",

~/.local/lib/python3.8/site-packages/modin/error_message.py in catch_bugs_and_request_email(cls, failure_condition, extra_log)
     49     def catch_bugs_and_request_email(cls, failure_condition, extra_log=""):
     50         if failure_condition:
---> 51             raise Exception(
     52                 "Internal Error. "
     53                 "Please email bug_reports@modin.org with the traceback and command that"

Exception: Internal Error. Please email bug_reports@modin.org with the traceback and command that caused this error.
Internal and external indices on axis 0 do not match.
```
@TK009 thank you this is very helpful.

It is worth investigating this because it does not seem like it should be throwing the error. I will re-open the issue so we can track it.

Resolving patch:
diff --git a/modin/backends/pandas/query_compiler.py b/modin/backends/pandas/query_compiler.py
--- a/modin/backends/pandas/query_compiler.py
+++ b/modin/backends/pandas/query_compiler.py
@@ -1559,6 +1559,20 @@ def describe(self, **kwargs):
             .astype(self.dtypes)
             .describe(**kwargs)
         )
+        new_index = empty_df.index
+
+        # Note: `describe` convert timestamp type to object type
+        # which results in the loss of two values in index: `first` and `last`
+        # for empty DataFrame.
+        datetime_is_numeric = kwargs.get("datetime_is_numeric") or False
+        if not any(map(is_numeric_dtype, empty_df.dtypes)) and not datetime_is_numeric:
+            for col_name in empty_df.dtypes.index:
+                # if previosly type of `col_name` was datetime or timedelta
+                if is_datetime_or_timedelta_dtype(self.dtypes[col_name]):
+                    new_index = pandas.Index(
+                        empty_df.index.to_list() + ["first"] + ["last"]
+                    )
+                    break
 
         def describe_builder(df, internal_indices=[]):
             return df.iloc[:, internal_indices].describe(**kwargs)
@@ -1568,7 +1582,7 @@ def describe_builder(df, internal_indices=[]):
                 0,
                 describe_builder,
                 empty_df.columns,
-                new_index=empty_df.index,
+                new_index=new_index,
                 new_columns=empty_df.columns,
             )
         )


Test patch:
diff --git a/modin/pandas/test/dataframe/test_reduction.py b/modin/pandas/test/dataframe/test_reduction.py
--- a/modin/pandas/test/dataframe/test_reduction.py
+++ b/modin/pandas/test/dataframe/test_reduction.py
@@ -140,6 +140,26 @@ def test_describe(data, percentiles):
     )
 
 
+@pytest.mark.parametrize("has_numeric_column", [False, True])
+@pytest.mark.parametrize("datetime_is_numeric", [True, False, None])
+def test_2195(datetime_is_numeric, has_numeric_column):
+    data = {
+        "categorical": pd.Categorical(["d"] * 10 ** 2),
+        "date": [np.datetime64("2000-01-01")] * 10 ** 2,
+    }
+
+    if has_numeric_column:
+        data.update({"numeric": [5] * 10 ** 2})
+
+    modin_df, pandas_df = pd.DataFrame(data), pandas.DataFrame(data)
+
+    eval_general(
+        modin_df,
+        pandas_df,
+        lambda df: df.describe(datetime_is_numeric=datetime_is_numeric),
+    )
+
+
 @pytest.mark.parametrize(
     "exclude,include",
     [