Repository: pandas-dev/pandas

Issue:
MultiIndex.remove_unused_levels() fills nans
#### Code Sample, a copy-pastable example if possible

```python
In [2]: mi = pd.MultiIndex.from_tuples([('a', 'w'), ('b', 'x'), (np.nan, 'y'), ('d', 'z')])

In [3]: list(mi) # OK
Out[3]: [('a', 'w'), ('b', 'x'), (nan, 'y'), ('d', 'z')]

In [4]: list(mi.remove_unused_levels()) # not OK
Out[4]: [('a', 'w'), ('b', 'x'), ('d', 'y'), ('d', 'z')]


```
#### Problem description

The missing value is replaced with another value.

#### Expected Output

``In [3]`` and ``In [4]`` should return the same.

#### Output of ``pd.show_versions()``

<details>

INSTALLED VERSIONS
------------------
commit: None
python: 3.5.3.final.0
python-bits: 64
OS: Linux
OS-release: 4.9.0-4-amd64
machine: x86_64
processor: 
byteorder: little
LC_ALL: None
LANG: en_GB.UTF-8
LOCALE: en_GB.UTF-8

pandas: 0.22.0.dev0+163.g3b05a60e6.dirty
pytest: 3.0.6
pip: 9.0.1
setuptools: 33.1.1
Cython: 0.25.2
numpy: 1.12.1
scipy: 0.18.1
pyarrow: None
xarray: None
IPython: 5.2.2
sphinx: None
patsy: 0.4.1+dev
dateutil: 2.6.0
pytz: 2016.10
blosc: None
bottleneck: 1.2.0
tables: 3.3.0
numexpr: 2.6.1
feather: 0.3.1
matplotlib: 2.0.0
openpyxl: 2.3.0
xlrd: 1.0.0
xlwt: 1.2.0
xlsxwriter: None
lxml: 3.7.1
bs4: 4.5.3
html5lib: 0.999999999
sqlalchemy: 1.0.15
pymysql: None
psycopg2: None
jinja2: 2.8
s3fs: None
fastparquet: None
pandas_gbq: None
pandas_datareader: None



</details>



Discussion:

Resolving patch:
diff --git a/pandas/core/indexes/multi.py b/pandas/core/indexes/multi.py
--- a/pandas/core/indexes/multi.py
+++ b/pandas/core/indexes/multi.py
@@ -1365,25 +1365,31 @@ def remove_unused_levels(self):
         new_labels = []
 
         changed = False
-        for lev, lab in zip(self.levels, self.labels):
+        for idx, (lev, lab) in enumerate(zip(self.levels, self.labels)):
+            na_idxs = np.where(lab == -1)[0]
+
+            if len(na_idxs):
+                lab = np.delete(lab, na_idxs)
 
             uniques = algos.unique(lab)
 
             # nothing unused
-            if len(uniques) == len(lev):
-                new_levels.append(lev)
-                new_labels.append(lab)
-                continue
+            if len(uniques) != len(lev):
+                changed = True
+
+                # labels get mapped from uniques to 0:len(uniques)
+                label_mapping = np.zeros(len(lev))
+                label_mapping[uniques] = np.arange(len(uniques))
 
-            changed = True
+                lab = label_mapping[lab]
 
-            # labels get mapped from uniques to 0:len(uniques)
-            label_mapping = np.zeros(len(lev))
-            label_mapping[uniques] = np.arange(len(uniques))
-            lab = label_mapping[lab]
+                # new levels are simple
+                lev = lev.take(uniques)
 
-            # new levels are simple
-            lev = lev.take(uniques)
+                if len(na_idxs):
+                    lab = np.insert(lab, na_idxs - np.arange(len(na_idxs)), -1)
+            else:
+                lab = self.labels[idx]
 
             new_levels.append(lev)
             new_labels.append(lab)


Test patch:
diff --git a/pandas/tests/indexes/test_multi.py b/pandas/tests/indexes/test_multi.py
--- a/pandas/tests/indexes/test_multi.py
+++ b/pandas/tests/indexes/test_multi.py
@@ -2629,6 +2629,20 @@ def test_reconstruct_remove_unused(self):
         tm.assert_index_equal(result2, expected)
         assert result2.is_(result)
 
+    @pytest.mark.parametrize('level0', [['a', 'd', 'b'],
+                                        ['a', 'd', 'b', 'unused']])
+    @pytest.mark.parametrize('level1', [['w', 'x', 'y', 'z'],
+                                        ['w', 'x', 'y', 'z', 'unused']])
+    def test_remove_unused_nan(self, level0, level1):
+        # GH 18417
+        mi = pd.MultiIndex(levels=[level0, level1],
+                           labels=[[0, 2, -1, 1, -1], [0, 1, 2, 3, 2]])
+
+        result = mi.remove_unused_levels()
+        tm.assert_index_equal(result, mi)
+        for level in 0, 1:
+            assert('unused' not in result.levels[level])
+
     @pytest.mark.parametrize('first_type,second_type', [
         ('int64', 'int64'),
         ('datetime64[D]', 'str')])