Repository: pandas-dev/pandas

Issue:
PERF: remove_unused_levels is very slow
#### Code Sample

```python
import numpy as np
import pandas as pd
series = pd.DataFrame(dict(
    A=np.random.randint(0, 10000, 100000),
    B=np.random.randint(0, 10000, 100000),
    V=np.random.rand(100000))).groupby(['A', 'B']).V.sum()
filtered_series = series[series < 0.1]
%time x = filtered_series.index.remove_unused_levels()
%time y = filtered_series.reset_index().set_index(['A', 'B']).index
```
#### Problem description

On my laptop, `x` takes 20 to 40 times as long as `y`, despite `y` doing the extra work of sorting the second level and reindexing the series in the process. The outputs, except for the sorting of the second level, are identical. Why is `remove_unused_levels` so slow?

#### Expected Output

`remove_unused_levels` should be at least as fast on large indexes as the `reset_index`/`set_index` hack.

#### Output of ``pd.show_versions()``
```
INSTALLED VERSIONS
------------------
commit: None
python: 3.6.1.final.0
python-bits: 64
OS: Linux
OS-release: 4.11.2-1-ARCH
machine: x86_64
processor: 
byteorder: little
LC_ALL: None
LANG: en_US.utf8
LOCALE: en_US.UTF-8

pandas: 0.20.1
pytest: None
pip: 9.0.1
setuptools: 35.0.2
Cython: None
numpy: 1.12.1
scipy: 0.19.0
xarray: None
IPython: 5.3.0
sphinx: None
patsy: None
dateutil: 2.6.0
pytz: 2017.2
blosc: None
bottleneck: None
tables: 3.4.2
numexpr: 2.6.2
feather: None
matplotlib: 2.0.0
openpyxl: None
xlrd: None
xlwt: None
xlsxwriter: None
lxml: None
bs4: 4.5.3
html5lib: None
sqlalchemy: None
pymysql: None
psycopg2: None
jinja2: 2.9.6
s3fs: None
pandas_gbq: None
pandas_datareader: None
```


Discussion:
look at my comment here: https://github.com/pandas-dev/pandas/blob/master/pandas/core/indexes/multi.py#L1314

you are welcome to provide another implementation if you'd like.
Cool, I may take a swing at that. Would a replacement implementation have to produce level lists with the same order as the current implementation, or does that not matter as long as `.values` remains the same as the input index? Docstring reads slightly ambiguously to me; it says "same .values and ordering", which I take to mean ordering of `.values` but might mean ordering of levels.
@rhendric 

I *think* you DO need to produce the same orderings, otherwise things won't be sorted. But not 100% sure this is a strict guarantee. We have *some* tests which exercise this (prob need a few more). 

The before and after *have* to be equal. So technically your example doesn't meet this test. However it may be possible to recompute the missing levels, then simply reorder them. (again the internals), not the ``.values``
Hm. Then I think there's not just a performance issue, but an actual bug. `filtered_series.index.remove_unused_levels().equals(filtered_series.index)` is false for me, when prepared as above.

Should I open a second issue or will this one serve for both?
no these won't compare equal
the level indexers have changed 
I think you're misreading me? I'm comparing the input and output of `remove_unused_levels` here, not comparing it with the different index from my original example.

Resolving patch:
diff --git a/asv_bench/benchmarks/indexing.py b/asv_bench/benchmarks/indexing.py
--- a/asv_bench/benchmarks/indexing.py
+++ b/asv_bench/benchmarks/indexing.py
@@ -204,6 +204,12 @@ def setup(self):
             [np.arange(100), list('A'), list('A')],
             names=['one', 'two', 'three'])
 
+        rng = np.random.RandomState(4)
+        size = 1 << 16
+        self.mi_unused_levels = pd.MultiIndex.from_arrays([
+            rng.randint(0, 1 << 13, size),
+            rng.randint(0, 1 << 10, size)])[rng.rand(size) < 0.1]
+
     def time_series_xs_mi_ix(self):
         self.s.ix[999]
 
@@ -248,6 +254,9 @@ def time_multiindex_small_get_loc_warm(self):
     def time_is_monotonic(self):
         self.miint.is_monotonic
 
+    def time_remove_unused_levels(self):
+        self.mi_unused_levels.remove_unused_levels()
+
 
 class IntervalIndexing(object):
     goal_time = 0.2
diff --git a/pandas/core/indexes/multi.py b/pandas/core/indexes/multi.py
--- a/pandas/core/indexes/multi.py
+++ b/pandas/core/indexes/multi.py
@@ -1290,8 +1290,8 @@ def remove_unused_levels(self):
         new_levels = []
         new_labels = []
 
-        changed = np.ones(self.nlevels, dtype=bool)
-        for i, (lev, lab) in enumerate(zip(self.levels, self.labels)):
+        changed = False
+        for lev, lab in zip(self.levels, self.labels):
 
             uniques = algos.unique(lab)
 
@@ -1299,33 +1299,29 @@ def remove_unused_levels(self):
             if len(uniques) == len(lev):
                 new_levels.append(lev)
                 new_labels.append(lab)
-                changed[i] = False
                 continue
 
-            # set difference, then reverse sort
-            diff = Index(np.arange(len(lev))).difference(uniques)
-            unused = diff.sort_values(ascending=False)
+            changed = True
+
+            # labels get mapped from uniques to 0:len(uniques)
+            label_mapping = np.zeros(len(lev))
+            label_mapping[uniques] = np.arange(len(uniques))
+            lab = label_mapping[lab]
 
             # new levels are simple
             lev = lev.take(uniques)
 
-            # new labels, we remove the unsued
-            # by decrementing the labels for that value
-            # prob a better way
-            for u in unused:
-
-                lab = np.where(lab > u, lab - 1, lab)
-
             new_levels.append(lev)
             new_labels.append(lab)
 
-        # nothing changed
-        if not changed.any():
-            return self
+        result = self._shallow_copy()
 
-        return MultiIndex(new_levels, new_labels,
-                          names=self.names, sortorder=self.sortorder,
-                          verify_integrity=False)
+        if changed:
+            result._reset_identity()
+            result._set_levels(new_levels, validate=False)
+            result._set_labels(new_labels, validate=False)
+
+        return result
 
     @property
     def nlevels(self):


Test patch:
diff --git a/pandas/tests/indexes/test_multi.py b/pandas/tests/indexes/test_multi.py
--- a/pandas/tests/indexes/test_multi.py
+++ b/pandas/tests/indexes/test_multi.py
@@ -2515,7 +2515,34 @@ def test_reconstruct_remove_unused(self):
         # idempotent
         result2 = result.remove_unused_levels()
         tm.assert_index_equal(result2, expected)
-        assert result2 is result
+        assert result2.is_(result)
+
+    @pytest.mark.parametrize('first_type,second_type', [
+        ('int64', 'int64'),
+        ('datetime64[D]', 'str')])
+    def test_remove_unused_levels_large(self, first_type, second_type):
+        # GH16556
+
+        # because tests should be deterministic (and this test in particular
+        # checks that levels are removed, which is not the case for every
+        # random input):
+        rng = np.random.RandomState(4)  # seed is arbitrary value that works
+
+        size = 1 << 16
+        df = DataFrame(dict(
+            first=rng.randint(0, 1 << 13, size).astype(first_type),
+            second=rng.randint(0, 1 << 10, size).astype(second_type),
+            third=rng.rand(size)))
+        df = df.groupby(['first', 'second']).sum()
+        df = df[df.third < 0.1]
+
+        result = df.index.remove_unused_levels()
+        assert len(result.levels[0]) < len(df.index.levels[0])
+        assert len(result.levels[1]) < len(df.index.levels[1])
+        assert result.equals(df.index)
+
+        expected = df.reset_index().set_index(['first', 'second']).index
+        tm.assert_index_equal(result, expected)
 
     def test_isin(self):
         values = [('foo', 2), ('bar', 3), ('quux', 4)]