Repository: pandas-dev/pandas

Issue:
PERF: unnecessary materialization of a MultiIndex.values when introspecting memory_usage
```
In [2]: import string
   ...: import pandas as pd
   ...: import numpy as np
   ...: 
   ...: def memory_usage(f):
   ...:     return f.memory_usage(deep=True).sum()
   ...: 
   ...: N = 100
   ...: M = len(string.uppercase)
   ...: df = pd.DataFrame({'value' : np.random.randn(N*M)},
   ...:                   index=pd.MultiIndex.from_product([list(string.uppercase),
   ...:                                                     pd.date_range('20160101',periods=N)],
   ...:                                                    names=['id','date'])
   ...:                   )
   ...: 
   ...: 
   ...: stacked = df.unstack('id')
   ...: 
   ...: assert df.values.nbytes == stacked.values.nbytes
   ...: 

In [3]: memory_usage(df)
Out[3]: 145600

In [4]: memory_usage(stacked)
Out[4]: 21600
I
n [7]: df.info(memory_usage='deep')
<class 'pandas.core.frame.DataFrame'>
MultiIndex: 2600 entries, (A, 2016-01-01 00:00:00) to (Z, 2016-04-09 00:00:00)
Data columns (total 1 columns):
value    2600 non-null float64
dtypes: float64(1)
memory usage: 142.2 KB

In [8]: stacked.info(memory_usage='deep')
<class 'pandas.core.frame.DataFrame'>
DatetimeIndex: 100 entries, 2016-01-01 to 2016-04-09
Freq: D
Data columns (total 26 columns):
(value, A)    100 non-null float64
(value, B)    100 non-null float64
(value, C)    100 non-null float64
(value, D)    100 non-null float64
(value, E)    100 non-null float64
(value, F)    100 non-null float64
(value, G)    100 non-null float64
(value, H)    100 non-null float64
(value, I)    100 non-null float64
(value, J)    100 non-null float64
(value, K)    100 non-null float64
(value, L)    100 non-null float64
(value, M)    100 non-null float64
(value, N)    100 non-null float64
(value, O)    100 non-null float64
(value, P)    100 non-null float64
(value, Q)    100 non-null float64
(value, R)    100 non-null float64
(value, S)    100 non-null float64
(value, T)    100 non-null float64
(value, U)    100 non-null float64
(value, V)    100 non-null float64
(value, W)    100 non-null float64
(value, X)    100 non-null float64
(value, Y)    100 non-null float64
(value, Z)    100 non-null float64
dtypes: float64(26)
memory usage: 21.1 KB
```

with this PR

```
In [2]: memory_usage(df)
Out[2]: 27088

In [3]: memory_usage(stacked)
Out[3]: 21600

In [4]: df.info(memory_usage='deep')
<class 'pandas.core.frame.DataFrame'>
MultiIndex: 2600 entries, (A, 2016-01-01 00:00:00) to (Z, 2016-04-09 00:00:00)
Data columns (total 1 columns):
value    2600 non-null float64
dtypes: float64(1)
memory usage: 26.5 KB

In [5]: stacked.info(memory_usage='deep')
<class 'pandas.core.frame.DataFrame'>
DatetimeIndex: 100 entries, 2016-01-01 to 2016-04-09
Freq: D
Data columns (total 26 columns):
(value, A)    100 non-null float64
(value, B)    100 non-null float64
(value, C)    100 non-null float64
(value, D)    100 non-null float64
(value, E)    100 non-null float64
(value, F)    100 non-null float64
(value, G)    100 non-null float64
(value, H)    100 non-null float64
(value, I)    100 non-null float64
(value, J)    100 non-null float64
(value, K)    100 non-null float64
(value, L)    100 non-null float64
(value, M)    100 non-null float64
(value, N)    100 non-null float64
(value, O)    100 non-null float64
(value, P)    100 non-null float64
(value, Q)    100 non-null float64
(value, R)    100 non-null float64
(value, S)    100 non-null float64
(value, T)    100 non-null float64
(value, U)    100 non-null float64
(value, V)    100 non-null float64
(value, W)    100 non-null float64
(value, X)    100 non-null float64
(value, Y)    100 non-null float64
(value, Z)    100 non-null float64
dtypes: float64(26)
memory usage: 21.1 KB
```



Discussion:

Resolving patch:
diff --git a/pandas/indexes/multi.py b/pandas/indexes/multi.py
--- a/pandas/indexes/multi.py
+++ b/pandas/indexes/multi.py
@@ -413,10 +413,27 @@ def _shallow_copy(self, values=None, **kwargs):
     def dtype(self):
         return np.dtype('O')
 
+    @Appender(Index.memory_usage.__doc__)
+    def memory_usage(self, deep=False):
+        # we are overwriting our base class to avoid
+        # computing .values here which could materialize
+        # a tuple representation uncessarily
+        return self._nbytes(deep)
+
     @cache_readonly
     def nbytes(self):
         """ return the number of bytes in the underlying data """
-        level_nbytes = sum((i.nbytes for i in self.levels))
+        return self._nbytes(False)
+
+    def _nbytes(self, deep=False):
+        """
+        return the number of bytes in the underlying data
+        deeply introspect the level data if deep=True
+
+        *this is in internal routine*
+
+        """
+        level_nbytes = sum((i.memory_usage(deep=deep) for i in self.levels))
         label_nbytes = sum((i.nbytes for i in self.labels))
         names_nbytes = sum((getsizeof(i) for i in self.names))
         return level_nbytes + label_nbytes + names_nbytes


Test patch:
diff --git a/pandas/tests/frame/test_repr_info.py b/pandas/tests/frame/test_repr_info.py
--- a/pandas/tests/frame/test_repr_info.py
+++ b/pandas/tests/frame/test_repr_info.py
@@ -381,3 +381,27 @@ def test_info_memory_usage(self):
         # deep=True, and add on some GC overhead
         diff = df.memory_usage(deep=True).sum() - sys.getsizeof(df)
         self.assertTrue(abs(diff) < 100)
+
+    def test_info_memory_usage_bug_on_multiindex(self):
+        # GH 14308
+        # memory usage introspection should not materialize .values
+
+        from string import ascii_uppercase as uppercase
+
+        def memory_usage(f):
+            return f.memory_usage(deep=True).sum()
+
+        N = 100
+        M = len(uppercase)
+        index = pd.MultiIndex.from_product([list(uppercase),
+                                            pd.date_range('20160101',
+                                                          periods=N)],
+                                           names=['id', 'date'])
+        df = DataFrame({'value': np.random.randn(N * M)}, index=index)
+
+        unstacked = df.unstack('id')
+        self.assertEqual(df.values.nbytes, unstacked.values.nbytes)
+        self.assertTrue(memory_usage(df) > memory_usage(unstacked))
+
+        # high upper bound
+        self.assertTrue(memory_usage(unstacked) - memory_usage(df) < 2000)