Repository: dask/dask

Issue:
Cannot reset index of dask.dataframe when underyling pandas Dataframes have UInt64Index multiIndex
Hellos, pretty much a duplicate of #2278 
except I'm noticing the issue only with `uint` dtypes

pandas==0.21.1
dask==0.14.3

Code:
```
import pandas as pd
import numpy as np
import dask.dataframe as dd
import dask
dask.set_options(get=dask.get)

df = pd.DataFrame(np.random.rand(2,3), columns=['A', 'B', 'C'])
df.A = df.A.astype(np.uint32)  # <<< any uint dtype
ddf = dd.from_pandas(df, npartitions=1)
grouped = ddf.groupby(['A', 'B']).agg({'C': 'sum'})
grouped = grouped.reset_index()
grouped.compute()
```

Traceback:
```
---------------------------------------------------------------------------
TypeError                                 Traceback (most recent call last)
/Users/jtam/anaconda2/envs/ipykernel_py3/lib/python3.6/site-packages/dask/dataframe/utils.py in raise_on_meta_error(funcname)
    157     try:
--> 158         yield
    159     except Exception as e:

/Users/jtam/anaconda2/envs/ipykernel_py3/lib/python3.6/site-packages/dask/dataframe/core.py in _emulate(func, *args, **kwargs)
   3086     with raise_on_meta_error(funcname(func)):
-> 3087         return func(*_extract_meta(args, True), **_extract_meta(kwargs, True))
   3088 

/Users/jtam/anaconda2/envs/ipykernel_py3/lib/python3.6/site-packages/dask/dataframe/core.py in _extract_meta(x, nonempty)
   3070     elif isinstance(x, tuple):
-> 3071         return tuple([_extract_meta(_x, nonempty) for _x in x])
   3072     elif isinstance(x, dict):

/Users/jtam/anaconda2/envs/ipykernel_py3/lib/python3.6/site-packages/dask/dataframe/core.py in <listcomp>(.0)
   3070     elif isinstance(x, tuple):
-> 3071         return tuple([_extract_meta(_x, nonempty) for _x in x])
   3072     elif isinstance(x, dict):

/Users/jtam/anaconda2/envs/ipykernel_py3/lib/python3.6/site-packages/dask/dataframe/core.py in _extract_meta(x, nonempty)
   3066         else:
-> 3067             return x._meta_nonempty if nonempty else x._meta
   3068     elif isinstance(x, list):

/Users/jtam/anaconda2/envs/ipykernel_py3/lib/python3.6/site-packages/dask/dataframe/core.py in _meta_nonempty(self)
    259         """ A non-empty version of `_meta` with fake data."""
--> 260         return meta_nonempty(self._meta)
    261 

/Users/jtam/anaconda2/envs/ipykernel_py3/lib/python3.6/site-packages/dask/dataframe/utils.py in meta_nonempty(x)
    425     elif isinstance(x, pd.DataFrame):
--> 426         idx = _nonempty_index(x.index)
    427         data = {i: _nonempty_series(x.iloc[:, i], idx)

/Users/jtam/anaconda2/envs/ipykernel_py3/lib/python3.6/site-packages/dask/dataframe/utils.py in _nonempty_index(idx)
    349     elif typ is pd.MultiIndex:
--> 350         levels = [_nonempty_index(i) for i in idx.levels]
    351         labels = [[0, 0] for i in idx.levels]

/Users/jtam/anaconda2/envs/ipykernel_py3/lib/python3.6/site-packages/dask/dataframe/utils.py in <listcomp>(.0)
    349     elif typ is pd.MultiIndex:
--> 350         levels = [_nonempty_index(i) for i in idx.levels]
    351         labels = [[0, 0] for i in idx.levels]

/Users/jtam/anaconda2/envs/ipykernel_py3/lib/python3.6/site-packages/dask/dataframe/utils.py in _nonempty_index(idx)
    353     raise TypeError("Don't know how to handle index of "
--> 354                     "type {0}".format(type(idx).__name__))
    355 

TypeError: Don't know how to handle index of type UInt64Index

During handling of the above exception, another exception occurred:

ValueError                                Traceback (most recent call last)
<ipython-input-23-02ee737c85ba> in <module>()
      8 df.A = df.A.astype(np.uint32)
      9 ddf = dd.from_pandas(df, npartitions=1)
---> 10 grouped = ddf.groupby(['A', 'B']).agg({'C': 'sum'})
     11 grouped = grouped.reset_index()
     12 grouped.compute()

/Users/jtam/anaconda2/envs/ipykernel_py3/lib/python3.6/site-packages/dask/dataframe/groupby.py in agg(self, arg, split_every, split_out)
   1007     @derived_from(pd.core.groupby.DataFrameGroupBy)
   1008     def agg(self, arg, split_every=None, split_out=1):
-> 1009         return self.aggregate(arg, split_every=split_every, split_out=split_out)
   1010 
   1011 

/Users/jtam/anaconda2/envs/ipykernel_py3/lib/python3.6/site-packages/dask/dataframe/groupby.py in aggregate(self, arg, split_every, split_out)
   1003             return self.size()
   1004 
-> 1005         return super(DataFrameGroupBy, self).aggregate(arg, split_every=split_every, split_out=split_out)
   1006 
   1007     @derived_from(pd.core.groupby.DataFrameGroupBy)

/Users/jtam/anaconda2/envs/ipykernel_py3/lib/python3.6/site-packages/dask/dataframe/groupby.py in aggregate(self, arg, split_every, split_out)
    884 
    885         return map_partitions(_agg_finalize, obj, token='aggregate-finalize',
--> 886                               funcs=finalizers)
    887 
    888     @insert_meta_param_description(pad=12)

/Users/jtam/anaconda2/envs/ipykernel_py3/lib/python3.6/site-packages/dask/dataframe/core.py in map_partitions(func, *args, **kwargs)
   3119 
   3120     if meta is no_default:
-> 3121         meta = _emulate(func, *args, **kwargs)
   3122 
   3123     if all(isinstance(arg, Scalar) for arg in args):

/Users/jtam/anaconda2/envs/ipykernel_py3/lib/python3.6/site-packages/dask/dataframe/core.py in _emulate(func, *args, **kwargs)
   3085     """
   3086     with raise_on_meta_error(funcname(func)):
-> 3087         return func(*_extract_meta(args, True), **_extract_meta(kwargs, True))
   3088 
   3089 

/Users/jtam/anaconda2/envs/ipykernel_py3/lib/python3.6/contextlib.py in __exit__(self, type, value, traceback)
     98                 value = type()
     99             try:
--> 100                 self.gen.throw(type, value, traceback)
    101                 raise RuntimeError("generator didn't stop after throw()")
    102             except StopIteration as exc:

/Users/jtam/anaconda2/envs/ipykernel_py3/lib/python3.6/site-packages/dask/dataframe/utils.py in raise_on_meta_error(funcname)
    169                ).format(" in `{0}`".format(funcname) if funcname else "",
    170                         repr(e), tb)
--> 171         raise ValueError(msg)
    172 
    173 

ValueError: Metadata inference failed in `_agg_finalize`.

Original error is below:
------------------------
TypeError("Don't know how to handle index of type UInt64Index",)

Traceback:
---------
  File "/Users/jtam/anaconda2/envs/ipykernel_py3/lib/python3.6/site-packages/dask/dataframe/utils.py", line 158, in raise_on_meta_error
    yield
  File "/Users/jtam/anaconda2/envs/ipykernel_py3/lib/python3.6/site-packages/dask/dataframe/core.py", line 3087, in _emulate
    return func(*_extract_meta(args, True), **_extract_meta(kwargs, True))
  File "/Users/jtam/anaconda2/envs/ipykernel_py3/lib/python3.6/site-packages/dask/dataframe/core.py", line 3071, in _extract_meta
    return tuple([_extract_meta(_x, nonempty) for _x in x])
  File "/Users/jtam/anaconda2/envs/ipykernel_py3/lib/python3.6/site-packages/dask/dataframe/core.py", line 3071, in <listcomp>
    return tuple([_extract_meta(_x, nonempty) for _x in x])
  File "/Users/jtam/anaconda2/envs/ipykernel_py3/lib/python3.6/site-packages/dask/dataframe/core.py", line 3067, in _extract_meta
    return x._meta_nonempty if nonempty else x._meta
  File "/Users/jtam/anaconda2/envs/ipykernel_py3/lib/python3.6/site-packages/dask/dataframe/core.py", line 260, in _meta_nonempty
    return meta_nonempty(self._meta)
  File "/Users/jtam/anaconda2/envs/ipykernel_py3/lib/python3.6/site-packages/dask/dataframe/utils.py", line 426, in meta_nonempty
    idx = _nonempty_index(x.index)
  File "/Users/jtam/anaconda2/envs/ipykernel_py3/lib/python3.6/site-packages/dask/dataframe/utils.py", line 350, in _nonempty_index
    levels = [_nonempty_index(i) for i in idx.levels]
  File "/Users/jtam/anaconda2/envs/ipykernel_py3/lib/python3.6/site-packages/dask/dataframe/utils.py", line 350, in <listcomp>
    levels = [_nonempty_index(i) for i in idx.levels]
  File "/Users/jtam/anaconda2/envs/ipykernel_py3/lib/python3.6/site-packages/dask/dataframe/utils.py", line 354, in _nonempty_index
    "type {0}".format(type(idx).__name__))
```


Discussion:
Thanks for the issue report. This is because the `UInt64Index` was recently added to pandas, and we don't have support for it yet. Will fix.

Resolving patch:
diff --git a/dask/dataframe/utils.py b/dask/dataframe/utils.py
--- a/dask/dataframe/utils.py
+++ b/dask/dataframe/utils.py
@@ -294,11 +294,17 @@ def make_meta(x, index=None):
     raise TypeError("Don't know how to create metadata from {0}".format(x))
 
 
+if PANDAS_VERSION >= "0.20.0":
+    _numeric_index_types = (pd.Int64Index, pd.Float64Index, pd.UInt64Index)
+else:
+    _numeric_index_types = (pd.Int64Index, pd.Float64Index)
+
+
 def _nonempty_index(idx):
     typ = type(idx)
     if typ is pd.RangeIndex:
         return pd.RangeIndex(2, name=idx.name)
-    elif typ in (pd.Int64Index, pd.Float64Index):
+    elif typ in _numeric_index_types:
         return typ([1, 2], name=idx.name)
     elif typ is pd.Index:
         return pd.Index(['a', 'b'], name=idx.name)


Test patch:
diff --git a/dask/dataframe/tests/test_utils_dataframe.py b/dask/dataframe/tests/test_utils_dataframe.py
--- a/dask/dataframe/tests/test_utils_dataframe.py
+++ b/dask/dataframe/tests/test_utils_dataframe.py
@@ -3,7 +3,8 @@
 import pandas.util.testing as tm
 import dask.dataframe as dd
 from dask.dataframe.utils import (shard_df_on_index, meta_nonempty, make_meta,
-                                  raise_on_meta_error, UNKNOWN_CATEGORIES)
+                                  raise_on_meta_error, UNKNOWN_CATEGORIES,
+                                  PANDAS_VERSION)
 
 import pytest
 
@@ -228,6 +229,15 @@ def test_meta_nonempty_index():
     assert res.names == idx.names
 
 
+@pytest.mark.skipif(PANDAS_VERSION < '0.20.0',
+                    reason="Pandas < 0.20.0 doesn't support UInt64Index")
+def test_meta_nonempty_uint64index():
+    idx = pd.UInt64Index([1], name='foo')
+    res = meta_nonempty(idx)
+    assert type(res) is pd.UInt64Index
+    assert res.name == idx.name
+
+
 def test_meta_nonempty_scalar():
     meta = meta_nonempty(np.float64(1.0))
     assert isinstance(meta, np.float64)