Repository: pandas-dev/pandas

Issue:
All-Nan MultiIndex level has different dtype than all-NaN flat Index
#### Code Sample, a copy-pastable example if possible

```python
In [3]: values = [np.nan, np.nan]

In [4]: pd.Index(values).dtype
Out[4]: dtype('float64')

In [5]: pd.MultiIndex.from_arrays([values]).levels[0].dtype
Out[5]: dtype('O')

In [6]: pd.MultiIndex.from_arrays([values, [2, 3]]).levels[0].dtype
Out[6]: dtype('O')
```

#### Problem description

Yes, I know, "who cares?". But this is biting me in fixing #17924.

#### Expected Output

The same - and I tend to think ``dtype('float64')`` is both preferred and more backwards-compatible.

#### Output of ``pd.show_versions()``

<details>

INSTALLED VERSIONS
------------------
commit: 51c5f4d2a20179041114fac0262ef567d0b0373f
python: 3.5.3.final.0
python-bits: 64
OS: Linux
OS-release: 4.9.0-3-amd64
machine: x86_64
processor: 
byteorder: little
LC_ALL: None
LANG: it_IT.UTF-8
LOCALE: it_IT.UTF-8

pandas: 0.21.0rc1+26.g51c5f4d2a.dirty
pytest: 3.0.6
pip: 9.0.1
setuptools: None
Cython: 0.25.2
numpy: 1.12.1
scipy: 0.19.0
pyarrow: None
xarray: None
IPython: 5.1.0.dev
sphinx: 1.5.6
patsy: 0.4.1
dateutil: 2.6.0
pytz: 2017.2
blosc: None
bottleneck: 1.2.1
tables: 3.3.0
numexpr: 2.6.1
feather: 0.3.1
matplotlib: 2.0.0
openpyxl: None
xlrd: 1.0.0
xlwt: 1.1.2
xlsxwriter: 0.9.6
lxml: None
bs4: 4.5.3
html5lib: 0.999999999
sqlalchemy: 1.0.15
pymysql: None
psycopg2: None
jinja2: 2.9.6
s3fs: None
fastparquet: None
pandas_gbq: None
pandas_datareader: 0.2.1


</details>



Discussion:
I am not sure this is possible to solve. NaNs are not really first class citizens in MIs, which means the level is actually empty, and the dtype of something empty is object. 
(at least this is the case for Index, not yet for Series https://github.com/pandas-dev/pandas/issues/17261)

> NaNs are not really first class citizens in MIs, which means the level is actually empty, 

Here I follow you

> and the dtype of something empty is object.

... here I don't: I think we should be able to have ``pd.MultiIndex.from_product([[], []])`` (0-length level) stored as empty ``object`` array but my example above (>0-length level with only missing values) stored as empy ``float`` array.
But how do you distinguish a 'real' empty MI or an MI with only NaNs ? As the actual `level` is the same for both: an empty index
> But how do you distinguish a 'real' empty MI or an MI with only NaNs ?

I would do it at initialization - see #17934 
```
In [13]: pd.MultiIndex.from_arrays([[pd.NaT, pd.NaT]]).levels[0]
Out[13]: DatetimeIndex([], dtype='datetime64[ns]', freq=None)

In [14]: pd.MultiIndex.from_arrays([[np.nan, np.nan]]).levels[0]
Out[14]: Index([], dtype='object')

In [15]: pd.MultiIndex.from_arrays([[None, None]]).levels[0]
Out[15]: Index([], dtype='object')
```

there *is* some ambiguity here, e.. whether [14] and [15] should match (since we use np,nan generically). Would be ok with making [14] float.

Resolving patch:
diff --git a/asv_bench/benchmarks/categoricals.py b/asv_bench/benchmarks/categoricals.py
--- a/asv_bench/benchmarks/categoricals.py
+++ b/asv_bench/benchmarks/categoricals.py
@@ -26,6 +26,9 @@ def setup(self):
         self.datetimes = pd.Series(pd.date_range(
             '1995-01-01 00:00:00', periods=10000, freq='s'))
 
+        self.values_some_nan = list(np.tile(self.categories + [np.nan], N))
+        self.values_all_nan = [np.nan] * len(self.values)
+
     def time_concat(self):
         concat([self.s, self.s])
 
@@ -46,6 +49,12 @@ def time_constructor_datetimes_with_nat(self):
         t.iloc[-1] = pd.NaT
         Categorical(t)
 
+    def time_constructor_with_nan(self):
+        Categorical(self.values_some_nan)
+
+    def time_constructor_all_nan(self):
+        Categorical(self.values_all_nan)
+
 
 class Categoricals2(object):
     goal_time = 0.2
diff --git a/pandas/core/categorical.py b/pandas/core/categorical.py
--- a/pandas/core/categorical.py
+++ b/pandas/core/categorical.py
@@ -288,6 +288,10 @@ def __init__(self, values, categories=None, ordered=None, dtype=None,
             self._dtype = dtype
             return
 
+        # null_mask indicates missing values we want to exclude from inference.
+        # This means: only missing values in list-likes (not arrays/ndframes).
+        null_mask = np.array(False)
+
         # sanitize input
         if is_categorical_dtype(values):
 
@@ -316,13 +320,14 @@ def __init__(self, values, categories=None, ordered=None, dtype=None,
             if not isinstance(values, np.ndarray):
                 values = _convert_to_list_like(values)
                 from pandas.core.series import _sanitize_array
-                # On list with NaNs, int values will be converted to float. Use
-                # "object" dtype to prevent this. In the end objects will be
-                # casted to int/... in the category assignment step.
-                if len(values) == 0 or isna(values).any():
+                # By convention, empty lists result in object dtype:
+                if len(values) == 0:
                     sanitize_dtype = 'object'
                 else:
                     sanitize_dtype = None
+                null_mask = isna(values)
+                if null_mask.any():
+                    values = [values[idx] for idx in np.where(~null_mask)[0]]
                 values = _sanitize_array(values, None, dtype=sanitize_dtype)
 
         if dtype.categories is None:
@@ -370,6 +375,12 @@ def __init__(self, values, categories=None, ordered=None, dtype=None,
                      "mean to use\n'Categorical.from_codes(codes, "
                      "categories)'?", RuntimeWarning, stacklevel=2)
 
+        if null_mask.any():
+            # Reinsert -1 placeholders for previously removed missing values
+            full_codes = - np.ones(null_mask.shape, dtype=codes.dtype)
+            full_codes[~null_mask] = codes
+            codes = full_codes
+
         self._dtype = dtype
         self._codes = coerce_indexer_dtype(codes, dtype.categories)
 


Test patch:
diff --git a/pandas/tests/indexes/test_multi.py b/pandas/tests/indexes/test_multi.py
--- a/pandas/tests/indexes/test_multi.py
+++ b/pandas/tests/indexes/test_multi.py
@@ -970,12 +970,13 @@ def test_get_level_values_na(self):
 
         arrays = [[np.nan, np.nan, np.nan], ['a', np.nan, 1]]
         index = pd.MultiIndex.from_arrays(arrays)
-        values = index.get_level_values(0)
-        expected = np.array([np.nan, np.nan, np.nan])
-        tm.assert_numpy_array_equal(values.values.astype(float), expected)
-        values = index.get_level_values(1)
-        expected = np.array(['a', np.nan, 1], dtype=object)
-        tm.assert_numpy_array_equal(values.values, expected)
+        result = index.get_level_values(0)
+        expected = pd.Index([np.nan, np.nan, np.nan])
+        tm.assert_index_equal(result, expected)
+
+        result = index.get_level_values(1)
+        expected = pd.Index(['a', np.nan, 1])
+        tm.assert_index_equal(result, expected)
 
         arrays = [['a', 'b', 'b'], pd.DatetimeIndex([0, 1, pd.NaT])]
         index = pd.MultiIndex.from_arrays(arrays)
diff --git a/pandas/tests/reshape/test_concat.py b/pandas/tests/reshape/test_concat.py
--- a/pandas/tests/reshape/test_concat.py
+++ b/pandas/tests/reshape/test_concat.py
@@ -648,7 +648,7 @@ def test_concat_categorical_coercion_nan(self):
         s1 = pd.Series([np.nan, np.nan], dtype='category')
         s2 = pd.Series([np.nan, np.nan])
 
-        exp = pd.Series([np.nan, np.nan, np.nan, np.nan], dtype=object)
+        exp = pd.Series([np.nan, np.nan, np.nan, np.nan])
         tm.assert_series_equal(pd.concat([s1, s2], ignore_index=True), exp)
         tm.assert_series_equal(s1.append(s2, ignore_index=True), exp)
         tm.assert_series_equal(pd.concat([s2, s1], ignore_index=True), exp)
diff --git a/pandas/tests/reshape/test_union_categoricals.py b/pandas/tests/reshape/test_union_categoricals.py
--- a/pandas/tests/reshape/test_union_categoricals.py
+++ b/pandas/tests/reshape/test_union_categoricals.py
@@ -90,7 +90,8 @@ def test_union_categoricals_nan(self):
         tm.assert_categorical_equal(res, exp)
 
         # all NaN
-        res = union_categoricals([pd.Categorical([np.nan, np.nan]),
+        res = union_categoricals([pd.Categorical(np.array([np.nan, np.nan],
+                                                          dtype=object)),
                                   pd.Categorical(['X'])])
         exp = Categorical([np.nan, np.nan, 'X'])
         tm.assert_categorical_equal(res, exp)
@@ -250,7 +251,7 @@ def test_union_categoricals_sort(self):
         c1 = Categorical([np.nan])
         c2 = Categorical([np.nan])
         result = union_categoricals([c1, c2], sort_categories=True)
-        expected = Categorical([np.nan, np.nan], categories=[])
+        expected = Categorical([np.nan, np.nan])
         tm.assert_categorical_equal(result, expected)
 
         c1 = Categorical([])
@@ -299,7 +300,7 @@ def test_union_categoricals_sort_false(self):
         c1 = Categorical([np.nan])
         c2 = Categorical([np.nan])
         result = union_categoricals([c1, c2], sort_categories=False)
-        expected = Categorical([np.nan, np.nan], categories=[])
+        expected = Categorical([np.nan, np.nan])
         tm.assert_categorical_equal(result, expected)
 
         c1 = Categorical([])