Repository: pandas-dev/pandas

Issue:
pd.MultiIndex.get_loc(np.nan)
#### Code Sample, a copy-pastable example if possible

```python
In [2]: mi = pd.MultiIndex.from_product([[np.nan, 1]] * 3)

In [3]: mi.get_loc(np.nan)
---------------------------------------------------------------------------
KeyError                                  Traceback (most recent call last)
/home/pietro/nobackup/repo/pandas/pandas/core/indexes/base.py in get_loc(self, key, method, tolerance)
   2513             try:
-> 2514                 return self._engine.get_loc(key)
   2515             except KeyError:

/home/pietro/nobackup/repo/pandas/pandas/_libs/index.pyx in pandas._libs.index.IndexEngine.get_loc (pandas/_libs/index.c:5834)()

/home/pietro/nobackup/repo/pandas/pandas/_libs/index.pyx in pandas._libs.index.IndexEngine.get_loc (pandas/_libs/index.c:5639)()

/home/pietro/nobackup/repo/pandas/pandas/_libs/index_class_helper.pxi in pandas._libs.index.Int64Engine._check_type (pandas/_libs/index.c:19008)()

KeyError: nan

During handling of the above exception, another exception occurred:

KeyError                                  Traceback (most recent call last)
<ipython-input-3-641c024055e9> in <module>()
----> 1 mi.get_loc(np.nan)

/home/pietro/nobackup/repo/pandas/pandas/core/indexes/multi.py in get_loc(self, key, method)
   2121 
   2122         if not isinstance(key, tuple):
-> 2123             loc = self._get_level_indexer(key, level=0)
   2124             return _maybe_to_slice(loc)
   2125 

/home/pietro/nobackup/repo/pandas/pandas/core/indexes/multi.py in _get_level_indexer(self, key, level, indexer)
   2411         else:
   2412 
-> 2413             loc = level_index.get_loc(key)
   2414             if isinstance(loc, slice):
   2415                 return loc

/home/pietro/nobackup/repo/pandas/pandas/core/indexes/base.py in get_loc(self, key, method, tolerance)
   2514                 return self._engine.get_loc(key)
   2515             except KeyError:
-> 2516                 return self._engine.get_loc(self._maybe_cast_indexer(key))
   2517 
   2518         indexer = self.get_indexer([key], method=method, tolerance=tolerance)

/home/pietro/nobackup/repo/pandas/pandas/_libs/index.pyx in pandas._libs.index.IndexEngine.get_loc (pandas/_libs/index.c:5834)()

/home/pietro/nobackup/repo/pandas/pandas/_libs/index.pyx in pandas._libs.index.IndexEngine.get_loc (pandas/_libs/index.c:5639)()

/home/pietro/nobackup/repo/pandas/pandas/_libs/index_class_helper.pxi in pandas._libs.index.Int64Engine._check_type (pandas/_libs/index.c:19008)()

KeyError: nan

```

#### Problem description

Opening a separate issue from #18485 (full label containing NaN) because, differently from that, this is _not_ fixed by #19074.

xref #5286 because they are probably worth testing together.

#### Expected Output

``slice(0, 4, None)``

#### Output of ``pd.show_versions()``

<details>

INSTALLED VERSIONS
------------------
commit: None
python: 3.5.3.final.0
python-bits: 64
OS: Linux
OS-release: 4.9.0-5-amd64
machine: x86_64
processor: 
byteorder: little
LC_ALL: None
LANG: en_GB.UTF-8
LOCALE: en_GB.UTF-8

pandas: 0.22.0.dev0+448.gdbec3c92e
pytest: 3.0.6
pip: 9.0.1
setuptools: 33.1.1
Cython: 0.25.2
numpy: 1.12.1
scipy: 0.18.1
pyarrow: None
xarray: None
IPython: 5.2.2
sphinx: None
patsy: 0.4.1+dev
dateutil: 2.6.0
pytz: 2016.10
blosc: None
bottleneck: 1.2.0
tables: 3.3.0
numexpr: 2.6.1
feather: 0.3.1
matplotlib: 2.0.0
openpyxl: 2.3.0
xlrd: 1.0.0
xlwt: 1.2.0
xlsxwriter: None
lxml: 3.7.1
bs4: 4.5.3
html5lib: 0.999999999
sqlalchemy: 1.0.15
pymysql: None
psycopg2: None
jinja2: 2.8
s3fs: None
fastparquet: None
pandas_gbq: None
pandas_datareader: None


</details>



Discussion:
yeah, nan handling is a bit of a problem. try this with multiple nan as well in say the 2nd level
  
I would like to take this issue if no one handle this.

Resolving patch:
diff --git a/pandas/core/indexes/multi.py b/pandas/core/indexes/multi.py
--- a/pandas/core/indexes/multi.py
+++ b/pandas/core/indexes/multi.py
@@ -2539,7 +2539,7 @@ def _partial_tup_index(self, tup, side="left"):
         for k, (lab, lev, labs) in enumerate(zipped):
             section = labs[start:end]
 
-            if lab not in lev:
+            if lab not in lev and not isna(lab):
                 if not lev.is_type_compatible(lib.infer_dtype([lab], skipna=False)):
                     raise TypeError(f"Level type mismatch: {lab}")
 
@@ -2549,13 +2549,38 @@ def _partial_tup_index(self, tup, side="left"):
                     loc -= 1
                 return start + section.searchsorted(loc, side=side)
 
-            idx = lev.get_loc(lab)
+            idx = self._get_loc_single_level_index(lev, lab)
             if k < n - 1:
                 end = start + section.searchsorted(idx, side="right")
                 start = start + section.searchsorted(idx, side="left")
             else:
                 return start + section.searchsorted(idx, side=side)
 
+    def _get_loc_single_level_index(self, level_index: Index, key: Hashable) -> int:
+        """
+        If key is NA value, location of index unify as -1.
+
+        Parameters
+        ----------
+        level_index: Index
+        key : label
+
+        Returns
+        -------
+        loc : int
+            If key is NA value, loc is -1
+            Else, location of key in index.
+
+        See Also
+        --------
+        Index.get_loc : The get_loc method for (single-level) index.
+        """
+
+        if is_scalar(key) and isna(key):
+            return -1
+        else:
+            return level_index.get_loc(key)
+
     def get_loc(self, key, method=None):
         """
         Get location for a label or a tuple of labels as an integer, slice or
@@ -2654,7 +2679,9 @@ def _maybe_to_slice(loc):
         loc = np.arange(start, stop, dtype="int64")
 
         for i, k in enumerate(follow_key, len(lead_key)):
-            mask = self.codes[i][loc] == self.levels[i].get_loc(k)
+            mask = self.codes[i][loc] == self._get_loc_single_level_index(
+                self.levels[i], k
+            )
             if not mask.all():
                 loc = loc[mask]
             if not len(loc):
@@ -2882,7 +2909,7 @@ def convert_indexer(start, stop, step, indexer=indexer, codes=level_codes):
 
         else:
 
-            code = level_index.get_loc(key)
+            code = self._get_loc_single_level_index(level_index, key)
 
             if level > 0 or self.lexsort_depth == 0:
                 # Desired level is not sorted
@@ -3377,14 +3404,11 @@ def isin(self, values, level=None):
             return algos.isin(self.values, values)
         else:
             num = self._get_level_number(level)
-            levs = self.levels[num]
-            level_codes = self.codes[num]
+            levs = self.get_level_values(num)
 
-            sought_labels = levs.isin(values).nonzero()[0]
             if levs.size == 0:
-                return np.zeros(len(level_codes), dtype=np.bool_)
-            else:
-                return np.lib.arraysetops.in1d(level_codes, sought_labels)
+                return np.zeros(len(levs), dtype=np.bool_)
+            return levs.isin(values)
 
 
 MultiIndex._add_numeric_methods_disabled()


Test patch:
diff --git a/pandas/tests/indexes/multi/test_contains.py b/pandas/tests/indexes/multi/test_contains.py
--- a/pandas/tests/indexes/multi/test_contains.py
+++ b/pandas/tests/indexes/multi/test_contains.py
@@ -98,3 +98,27 @@ def test_isin_level_kwarg():
 
     with pytest.raises(KeyError, match="'Level C not found'"):
         idx.isin(vals_1, level="C")
+
+
+def test_contains_with_missing_value():
+    # issue 19132
+    idx = MultiIndex.from_arrays([[1, np.nan, 2]])
+    assert np.nan in idx
+
+    idx = MultiIndex.from_arrays([[1, 2], [np.nan, 3]])
+    assert np.nan not in idx
+    assert (1, np.nan) in idx
+
+
+@pytest.mark.parametrize(
+    "labels,expected,level",
+    [
+        ([("b", np.nan)], np.array([False, False, True]), None,),
+        ([np.nan, "a"], np.array([True, True, False]), 0),
+        (["d", np.nan], np.array([False, True, True]), 1),
+    ],
+)
+def test_isin_multi_index_with_missing_value(labels, expected, level):
+    # GH 19132
+    midx = MultiIndex.from_arrays([[np.nan, "a", "b"], ["c", "d", np.nan]])
+    tm.assert_numpy_array_equal(midx.isin(labels, level=level), expected)
diff --git a/pandas/tests/indexes/multi/test_indexing.py b/pandas/tests/indexes/multi/test_indexing.py
--- a/pandas/tests/indexes/multi/test_indexing.py
+++ b/pandas/tests/indexes/multi/test_indexing.py
@@ -437,3 +437,91 @@ def test_timestamp_multiindex_indexer():
     )
     should_be = pd.Series(data=np.arange(24, len(qidx) + 24), index=qidx, name="foo")
     tm.assert_series_equal(result, should_be)
+
+
+def test_get_loc_with_values_including_missing_values():
+    # issue 19132
+    idx = MultiIndex.from_product([[np.nan, 1]] * 2)
+    expected = slice(0, 2, None)
+    assert idx.get_loc(np.nan) == expected
+
+    idx = MultiIndex.from_arrays([[np.nan, 1, 2, np.nan]])
+    expected = np.array([True, False, False, True])
+    tm.assert_numpy_array_equal(idx.get_loc(np.nan), expected)
+
+    idx = MultiIndex.from_product([[np.nan, 1]] * 3)
+    expected = slice(2, 4, None)
+    assert idx.get_loc((np.nan, 1)) == expected
+
+
+@pytest.mark.parametrize(
+    "index_arr,labels,expected",
+    [
+        (
+            [[1, np.nan, 2], [3, 4, 5]],
+            [1, np.nan, 2],
+            np.array([-1, -1, -1], dtype=np.intp),
+        ),
+        ([[1, np.nan, 2], [3, 4, 5]], [(np.nan, 4)], np.array([1], dtype=np.intp)),
+        ([[1, 2, 3], [np.nan, 4, 5]], [(1, np.nan)], np.array([0], dtype=np.intp)),
+        (
+            [[1, 2, 3], [np.nan, 4, 5]],
+            [np.nan, 4, 5],
+            np.array([-1, -1, -1], dtype=np.intp),
+        ),
+    ],
+)
+def test_get_indexer_with_missing_value(index_arr, labels, expected):
+    # issue 19132
+    idx = MultiIndex.from_arrays(index_arr)
+    result = idx.get_indexer(labels)
+    tm.assert_numpy_array_equal(result, expected)
+
+
+@pytest.mark.parametrize(
+    "index_arr,expected,target,algo",
+    [
+        ([[np.nan, "a", "b"], ["c", "d", "e"]], 0, np.nan, "left"),
+        ([[np.nan, "a", "b"], ["c", "d", "e"]], 1, (np.nan, "c"), "right"),
+        ([["a", "b", "c"], ["d", np.nan, "d"]], 1, ("b", np.nan), "left"),
+    ],
+)
+def test_get_slice_bound_with_missing_value(index_arr, expected, target, algo):
+    # issue 19132
+    idx = MultiIndex.from_arrays(index_arr)
+    result = idx.get_slice_bound(target, side=algo, kind="loc")
+    assert result == expected
+
+
+@pytest.mark.parametrize(
+    "index_arr,expected,start_idx,end_idx",
+    [
+        ([[np.nan, 1, 2], [3, 4, 5]], slice(0, 2, None), np.nan, 1),
+        ([[np.nan, 1, 2], [3, 4, 5]], slice(0, 3, None), np.nan, (2, 5)),
+        ([[1, 2, 3], [4, np.nan, 5]], slice(1, 3, None), (2, np.nan), 3),
+        ([[1, 2, 3], [4, np.nan, 5]], slice(1, 3, None), (2, np.nan), (3, 5)),
+    ],
+)
+def test_slice_indexer_with_missing_value(index_arr, expected, start_idx, end_idx):
+    # issue 19132
+    idx = MultiIndex.from_arrays(index_arr)
+    result = idx.slice_indexer(start=start_idx, end=end_idx)
+    assert result == expected
+
+
+@pytest.mark.parametrize(
+    "index_arr,expected,start_idx,end_idx",
+    [
+        ([[np.nan, "a", "b"], ["c", "d", "e"]], (0, 3), np.nan, None),
+        ([[np.nan, "a", "b"], ["c", "d", "e"]], (0, 3), np.nan, "b"),
+        ([[np.nan, "a", "b"], ["c", "d", "e"]], (0, 3), np.nan, ("b", "e")),
+        ([["a", "b", "c"], ["d", np.nan, "e"]], (1, 3), ("b", np.nan), None),
+        ([["a", "b", "c"], ["d", np.nan, "e"]], (1, 3), ("b", np.nan), "c"),
+        ([["a", "b", "c"], ["d", np.nan, "e"]], (1, 3), ("b", np.nan), ("c", "e")),
+    ],
+)
+def test_slice_locs_with_missing_value(index_arr, expected, start_idx, end_idx):
+    # issue 19132
+    idx = MultiIndex.from_arrays(index_arr)
+    result = idx.slice_locs(start=start_idx, end=end_idx)
+    assert result == expected