Repository: modin-project/modin

Issue:
Internal Error when running memory_usage()
### 🤖 This is a bot message 🤖 

bug_reports@modin.org has been sent an email reporting the following bug:

```
Traceback (most recent call last):
  File "modin_operations.py", line 39, in <module>
    measure_operations(args.mode)
  File "modin_operations.py", line 31, in measure_operations
    measure_time(read_operation, pd, filename)
  File "modin_operations.py", line 7, in measure_time
    ret_val = func(*args)
  File "modin_operations.py", line 15, in read_operation
    print(df.memory_usage())
  File "/Users/krutivanatwala/Documents/python-experiments/venv/lib/python3.6/site-packages/modin/pandas/base.py", line 3263, in __str__
    return repr(self)
  File "/Users/krutivanatwala/Documents/python-experiments/venv/lib/python3.6/site-packages/modin/pandas/series.py", line 213, in __repr__
    temp_df = self._build_repr_df(num_rows, num_cols)
  File "/Users/krutivanatwala/Documents/python-experiments/venv/lib/python3.6/site-packages/modin/pandas/base.py", line 61, in _build_repr_df
    head_front = head.to_pandas()
  File "/Users/krutivanatwala/Documents/python-experiments/venv/lib/python3.6/site-packages/modin/backends/pandas/query_compiler.py", line 487, in to_pandas
    len(df.index) != len(self.index) or len(df.columns) != len(self.columns)
  File "/Users/krutivanatwala/Documents/python-experiments/venv/lib/python3.6/site-packages/modin/error_message.py", line 38, in catch_bugs_and_request_email
    "Internal Error. "
Exception: Internal Error. Please email bug_reports@modin.org with the traceback and command that caused this error.
```

Created a dataset as follows :
```
import numpy as np
import pandas as pd
df = pd.DataFrame(np.random.randint(0, 100, size=(10000000, 4)), columns=list('ABCD'))
df.to_csv('test_modin.csv', index=False)
```
Then was trying to read it as:
```
df = pd.read_csv(f)
print(df.memory_usage())
```


Discussion:
I can reproduce this on master.

Resolving patch:
diff --git a/modin/backends/pandas/query_compiler.py b/modin/backends/pandas/query_compiler.py
--- a/modin/backends/pandas/query_compiler.py
+++ b/modin/backends/pandas/query_compiler.py
@@ -840,7 +840,7 @@ def transpose(self, *args, **kwargs):
             Transposed new DataManager.
         """
         new_data = self.data.transpose(*args, **kwargs)
-        # Switch the index and columns and transpose the
+        # Switch the index and columns and transpose the data within the blocks.
         new_manager = self.__constructor__(
             new_data, self.columns, self.index, is_transposed=self._is_transposed ^ 1
         )
@@ -1277,8 +1277,12 @@ def memory_usage(self, **kwargs):
         def memory_usage_builder(df, **kwargs):
             return df.memory_usage(**kwargs)
 
-        func = self._build_mapreduce_func(memory_usage_builder, **kwargs)
-        return self._full_axis_reduce(0, func)
+        def sum_memory_usage(df):
+            return df.sum()
+
+        map_func = self._build_mapreduce_func(memory_usage_builder, **kwargs)
+        reduce_func = self._build_mapreduce_func(sum_memory_usage)
+        return self._full_reduce(0, map_func, reduce_func)
 
     def nunique(self, **kwargs):
         """Returns the number of unique items over each column or row.
diff --git a/modin/engines/base/frame/partition_manager.py b/modin/engines/base/frame/partition_manager.py
--- a/modin/engines/base/frame/partition_manager.py
+++ b/modin/engines/base/frame/partition_manager.py
@@ -55,10 +55,8 @@ def __constructor__(self):
     _filtered_empties = False
 
     def _get_partitions(self):
-        if (
-            not self._filtered_empties
-            and self._lengths_cache is not None
-            and self._widths_cache is not None
+        if not self._filtered_empties or (
+            self._lengths_cache is not None and self._widths_cache is not None
         ):
             self._partitions_cache = np.array(
                 [
diff --git a/modin/pandas/series.py b/modin/pandas/series.py
--- a/modin/pandas/series.py
+++ b/modin/pandas/series.py
@@ -207,7 +207,6 @@ def __pow__(self, right):
         return self.pow(right)
 
     def __repr__(self):
-        # In the future, we can have this be configurable, just like Pandas.
         num_rows = pandas.get_option("max_rows") or 60
         num_cols = pandas.get_option("max_columns") or 20
         temp_df = self._build_repr_df(num_rows, num_cols)


Test patch:
diff --git a/modin/pandas/test/test_dataframe.py b/modin/pandas/test/test_dataframe.py
--- a/modin/pandas/test/test_dataframe.py
+++ b/modin/pandas/test/test_dataframe.py
@@ -3061,15 +3061,17 @@ def test_melt(self):
             pd.DataFrame(data).melt()
 
     @pytest.mark.parametrize("data", test_data_values, ids=test_data_keys)
-    def test_memory_usage(self, data):
+    @pytest.mark.parametrize(
+        "index", bool_arg_values, ids=arg_keys("index", bool_arg_keys)
+    )
+    def test_memory_usage(self, data, index):
         modin_df = pd.DataFrame(data)
         pandas_df = pandas.DataFrame(data)  # noqa F841
 
-        assert modin_df.memory_usage(index=True).at["Index"] is not None
-        assert (
-            modin_df.memory_usage(deep=True).sum()
-            >= modin_df.memory_usage(deep=False).sum()
-        )
+        modin_result = modin_df.memory_usage(index=index)
+        pandas_result = pandas_df.memory_usage(index=index)
+
+        df_equals(modin_result, pandas_result)
 
     def test_merge(self):
         frame_data = {