Repository: dask/dask

Issue:
DataFrame.groupby.mean fails with multiple categoricals
Seems closely related to issue #2510, but it looks like there's still a problem with `groupby.mean`.  Here's a quick example:

```
import dask.dataframe as dd

dt = {"col1": "category",
      "col2": "category",
      "col3": int}
df = dd.read_csv("data.csv", dtype=dt)

df.compute()
#   col1 col2  col3
# 0    a    y     0
# 1    a    z     1
# 2    b    y     0
# 3    b    z     1

# Gives an error
df.groupby(by=["col1", "col2"])["col3"].mean().compute()
# ValueError: Level values must be unique: [nan, nan] on level 0

# Taking a sum works
df.groupby(by=["col1", "col2"])["col3"].sum().compute()
# col1  col2
# a     y       0
#       z       1
# b     y       0
#       z       1
# Name: col3, dtype: int64

# Grouping by only one column also works
df.groupby(by="col1")["col3"].mean().compute()
# col1
# a    0.5
# b    0.5
# Name: col3, dtype: float64

# Use object dtype instead
df = dd.read_csv("data.csv")
df.dtypes
# col1    object
# col2    object
# col3     int64
# dtype: object

# Works now
df.groupby(by=["col1", "col2"]).mean().compute()
#            col3
# col1 col2      
# a    y      0.0
#      z      1.0
# b    y      0.0
#      z      1.0
```

This is `dask` 0.17.5 and `pandas` 0.24.0.


Discussion:
Thanks for the bug report @dsaxton .  If possible I recommend two steps:

1.   Verify that this error also occurs with the latest release of dask
2.  Try to produce a minimal example.  See http://matthewrocklin.com/blog/work/2018/02/28/minimal-bug-reports for more information
Here's an example using version 1.1.1 with the full stack trace:

```
import pandas as pd 
import dask.dataframe as dd 
 
data = {"col1": ["a", "a", "b", "b"], 
        "col2": ["y", "z", "y", "z"], 
        "col3": [0, 1, 0, 1]} 
 
df = dd.from_pandas(pd.DataFrame(data), chunksize=2) 
df[["col1", "col2"]] = df[["col1", "col2"]].astype("category") 
 
df.groupby(by=["col1", "col2"])["col3"].mean().compute() 

---------------------------------------------------------------------------
ValueError                                Traceback (most recent call last)
<ipython-input-1-53ba1bf325bc> in <module>
      9 df[["col1", "col2"]] = df[["col1", "col2"]].astype("category")
     10 
---> 11 df.groupby(by=["col1", "col2"])["col3"].mean().compute()

/anaconda3/lib/python3.6/site-packages/dask/dataframe/groupby.py in mean(self, split_every, split_out)
    925     def mean(self, split_every=None, split_out=1):
    926         return (self.sum(split_every=split_every, split_out=split_out) /
--> 927                 self.count(split_every=split_every, split_out=split_out))
    928 
    929     @derived_from(pd.core.groupby.GroupBy)

/anaconda3/lib/python3.6/site-packages/dask/dataframe/core.py in <lambda>(self, other)
   1198             return lambda self, other: elemwise(op, other, self)
   1199         else:
-> 1200             return lambda self, other: elemwise(op, self, other)
   1201 
   1202     def rolling(self, window, min_periods=None, freq=None, center=False,

/anaconda3/lib/python3.6/site-packages/dask/dataframe/core.py in elemwise(op, *args, **kwargs)
   3409         # For broadcastable series, use no rows.
   3410         parts = [d._meta if _is_broadcastable(d) or isinstance(d, Array)
-> 3411                  else d._meta_nonempty for d in dasks]
   3412         with raise_on_meta_error(funcname(op)):
   3413             meta = partial_by_order(*parts, function=op, other=other)

/anaconda3/lib/python3.6/site-packages/dask/dataframe/core.py in <listcomp>(.0)
   3409         # For broadcastable series, use no rows.
   3410         parts = [d._meta if _is_broadcastable(d) or isinstance(d, Array)
-> 3411                  else d._meta_nonempty for d in dasks]
   3412         with raise_on_meta_error(funcname(op)):
   3413             meta = partial_by_order(*parts, function=op, other=other)

/anaconda3/lib/python3.6/site-packages/dask/dataframe/core.py in _meta_nonempty(self)
    305     def _meta_nonempty(self):
    306         """ A non-empty version of `_meta` with fake data."""
--> 307         return meta_nonempty(self._meta)
    308 
    309     @property

/anaconda3/lib/python3.6/site-packages/dask/utils.py in __call__(self, arg, *args, **kwargs)
    411         """
    412         meth = self.dispatch(type(arg))
--> 413         return meth(arg, *args, **kwargs)
    414 
    415     @property

/anaconda3/lib/python3.6/site-packages/dask/dataframe/utils.py in _nonempty_series(s, idx)
    453     # TODO: Use register dtypes with make_array_nonempty
    454     if idx is None:
--> 455         idx = _nonempty_index(s.index)
    456     dtype = s.dtype
    457     if is_datetime64tz_dtype(dtype):

/anaconda3/lib/python3.6/site-packages/dask/dataframe/utils.py in _nonempty_index(idx)
    409         levels = [_nonempty_index(l) for l in idx.levels]
    410         labels = [[0, 0] for i in idx.levels]
--> 411         return pd.MultiIndex(levels=levels, labels=labels, names=idx.names)
    412     raise TypeError("Don't know how to handle index of "
    413                     "type {0}".format(type(idx).__name__))

/anaconda3/lib/python3.6/site-packages/pandas/core/indexes/multi.py in __new__(cls, levels, labels, sortorder, names, dtype, copy, name, verify_integrity, _set_identity)
    240 
    241         if verify_integrity:
--> 242             result._verify_integrity()
    243         if _set_identity:
    244             result._reset_identity()

/anaconda3/lib/python3.6/site-packages/pandas/core/indexes/multi.py in _verify_integrity(self, labels, levels)
    283                                  "level {level}".format(
    284                                      values=[value for value in level],
--> 285                                      level=i))
    286 
    287     @property

ValueError: Level values must be unique: [nan, nan] on level 0
```
Thanks for the excellent reproducible example @dsaxton !

cc @TomAugspurger in case he's able to interpret the error message

Resolving patch:
diff --git a/dask/dataframe/utils.py b/dask/dataframe/utils.py
--- a/dask/dataframe/utils.py
+++ b/dask/dataframe/utils.py
@@ -398,13 +398,12 @@ def _nonempty_index(idx):
                                  name=idx.name)
     elif typ is pd.CategoricalIndex:
         if len(idx.categories) == 0:
-            data = _nonempty_index(idx.categories)
-            cats = None
+            data = pd.Categorical(_nonempty_index(idx.categories),
+                                  ordered=idx.ordered)
         else:
-            data = _nonempty_index(_nonempty_index(idx.categories))
-            cats = idx.categories
-        return pd.CategoricalIndex(data, categories=cats,
-                                   ordered=idx.ordered, name=idx.name)
+            data = pd.Categorical.from_codes(
+                [-1, 0], categories=idx.categories, ordered=idx.ordered)
+        return pd.CategoricalIndex(data, name=idx.name)
     elif typ is pd.MultiIndex:
         levels = [_nonempty_index(l) for l in idx.levels]
         labels = [[0, 0] for i in idx.levels]


Test patch:
diff --git a/dask/dataframe/tests/test_utils_dataframe.py b/dask/dataframe/tests/test_utils_dataframe.py
--- a/dask/dataframe/tests/test_utils_dataframe.py
+++ b/dask/dataframe/tests/test_utils_dataframe.py
@@ -209,7 +209,7 @@ def test_meta_nonempty_index():
     assert res.freq == idx.freq
     assert res.name == idx.name
 
-    idx = pd.CategoricalIndex(['a'], ['a', 'b'], ordered=True, name='foo')
+    idx = pd.CategoricalIndex(['xyx'], ['xyx', 'zzz'], ordered=True, name='foo')
     res = meta_nonempty(idx)
     assert type(res) is pd.CategoricalIndex
     assert (res.categories == idx.categories).all()
@@ -233,7 +233,7 @@ def test_meta_nonempty_index():
     assert res.names == idx.names
 
     levels = [pd.Int64Index([1], name='a'),
-              pd.CategoricalIndex(data=['b'], categories=['b'], name='b'),
+              pd.CategoricalIndex(data=['xyx'], categories=['xyx'], name='b'),
               pd.TimedeltaIndex([np.timedelta64(1, 'D')], name='timedelta')]
     idx = pd.MultiIndex(levels=levels, labels=[[0], [0], [0]], names=['a', 'b', 'timedelta'])
     res = meta_nonempty(idx)