Repository: dask/dask

Issue:
DataFrame.groupby with multiple non-identical Categoricals
xref #1877 - this works fine in the single column case, but not when grouping by multiple Categoricals.

```
import pandas as pd
import dask.dataframe as dd

In [41]: a = pd.DataFrame({'c1': pd.Categorical(['a', 'b', 'c']), 
    ...:                   'c2': pd.Categorical(['a', 'b', 'c']),
    ...:                   'v': [1, 2, 3]})

In [42]: b = pd.DataFrame({'c1': pd.Categorical(['a', 'b', 'd']), 
    ...:                   'c2': pd.Categorical(['a', 'b', 'd']),
    ...:                   'v': [4, 5, 6]})

In [44]: from dask import delayed

In [45]: ddf = dd.from_delayed([delayed(a), delayed(b)])

In [47]: ddf.groupby('c1')['v'].sum().compute()
Out[47]: 
c1
a    5
b    7
c    3
d    6
Name: v, dtype: int64

In [49]: ddf.groupby(['c1','c2'])['v'].sum().compute()
---------------------------------------------------------------------------
TypeError                                 Traceback (most recent call last)
<ipython-input-49-d712b35f9c30> in <module>()
----> 1 ddf.groupby(['c1','c2'])['v'].sum().compute()

~\AppData\Local\Continuum\Anaconda3\lib\site-packages\dask\base.py in compute(self, **kwargs)
     95             Extra keywords to forward to the scheduler ``get`` function.
     96         """
---> 97         (result,) = compute(self, traverse=False, **kwargs)
     98         return result
     99 

~\AppData\Local\Continuum\Anaconda3\lib\site-packages\dask\base.py in compute(*args, **kwargs)
    202     dsk = collections_to_dsk(variables, optimize_graph, **kwargs)
    203     keys = [var._keys() for var in variables]
--> 204     results = get(dsk, keys, **kwargs)
    205 
    206     results_iter = iter(results)

~\AppData\Local\Continuum\Anaconda3\lib\site-packages\dask\threaded.py in get(dsk, result, cache, num_workers, **kwargs)
     73     results = get_async(pool.apply_async, len(pool._pool), dsk, result,
     74                         cache=cache, get_id=_thread_get_id,
---> 75                         pack_exception=pack_exception, **kwargs)
     76 
     77     # Cleanup pools associated to dead threads

~\AppData\Local\Continuum\Anaconda3\lib\site-packages\dask\local.py in get_async(apply_async, num_workers, dsk, result, cache, get_id, rerun_exceptions_locally, pack_exception, raise_exception, callbacks, dumps, loads, **kwargs)
    519                         _execute_task(task, data)  # Re-execute locally
    520                     else:
--> 521                         raise_exception(exc, tb)
    522                 res, worker_id = loads(res_info)
    523                 state['cache'][key] = res

~\AppData\Local\Continuum\Anaconda3\lib\site-packages\dask\compatibility.py in reraise(exc, tb)
     58         if exc.__traceback__ is not tb:
     59             raise exc.with_traceback(tb)
---> 60         raise exc
     61 
     62 else:

~\AppData\Local\Continuum\Anaconda3\lib\site-packages\dask\local.py in execute_task(key, task_info, dumps, loads, get_id, pack_exception)
    288     try:
    289         task, data = loads(task_info)
--> 290         result = _execute_task(task, data)
    291         id = get_id()
    292         result = dumps((result, id))

~\AppData\Local\Continuum\Anaconda3\lib\site-packages\dask\local.py in _execute_task(arg, cache, dsk)
    268     elif istask(arg):
    269         func, args = arg[0], arg[1:]
--> 270         args2 = [_execute_task(a, cache) for a in args]
    271         return func(*args2)
    272     elif not ishashable(arg):

~\AppData\Local\Continuum\Anaconda3\lib\site-packages\dask\local.py in <listcomp>(.0)
    268     elif istask(arg):
    269         func, args = arg[0], arg[1:]
--> 270         args2 = [_execute_task(a, cache) for a in args]
    271         return func(*args2)
    272     elif not ishashable(arg):

~\AppData\Local\Continuum\Anaconda3\lib\site-packages\dask\local.py in _execute_task(arg, cache, dsk)
    265     """
    266     if isinstance(arg, list):
--> 267         return [_execute_task(a, cache) for a in arg]
    268     elif istask(arg):
    269         func, args = arg[0], arg[1:]

~\AppData\Local\Continuum\Anaconda3\lib\site-packages\dask\local.py in <listcomp>(.0)
    265     """
    266     if isinstance(arg, list):
--> 267         return [_execute_task(a, cache) for a in arg]
    268     elif istask(arg):
    269         func, args = arg[0], arg[1:]

~\AppData\Local\Continuum\Anaconda3\lib\site-packages\dask\local.py in _execute_task(arg, cache, dsk)
    269         func, args = arg[0], arg[1:]
    270         args2 = [_execute_task(a, cache) for a in args]
--> 271         return func(*args2)
    272     elif not ishashable(arg):
    273         return arg

~\AppData\Local\Continuum\Anaconda3\lib\site-packages\dask\dataframe\core.py in _concat(args)
     59     # this seems easier. TODO: don't do this.
     60     args2 = [i for i in args if len(i)]
---> 61     return args[0] if not args2 else methods.concat(args2, uniform=True)
     62 
     63 

~\AppData\Local\Continuum\Anaconda3\lib\site-packages\dask\dataframe\methods.py in concat(dfs, axis, join, uniform)
    296             return pd.Series(union_categoricals(dfs2), index=ind,
    297                              name=dfs2[0].name)
--> 298         out = pd.concat(dfs2, join=join)
    299     # Re-add the index if needed
    300     if ind is not None:

~\AppData\Local\Continuum\Anaconda3\lib\site-packages\pandas\core\reshape\concat.py in concat(objs, axis, join, join_axes, ignore_index, keys, levels, names, verify_integrity, copy)
    204                        keys=keys, levels=levels, names=names,
    205                        verify_integrity=verify_integrity,
--> 206                        copy=copy)
    207     return op.get_result()
    208 

~\AppData\Local\Continuum\Anaconda3\lib\site-packages\pandas\core\reshape\concat.py in __init__(self, objs, axis, join, join_axes, keys, levels, names, ignore_index, verify_integrity, copy)
    352         self.copy = copy
    353 
--> 354         self.new_axes = self._get_new_axes()
    355 
    356     def get_result(self):

~\AppData\Local\Continuum\Anaconda3\lib\site-packages\pandas\core\reshape\concat.py in _get_new_axes(self)
    440                 new_axes[i] = ax
    441 
--> 442         new_axes[self.axis] = self._get_concat_axis()
    443         return new_axes
    444 

~\AppData\Local\Continuum\Anaconda3\lib\site-packages\pandas\core\reshape\concat.py in _get_concat_axis(self)
    494 
    495         if self.keys is None:
--> 496             concat_axis = _concat_indexes(indexes)
    497         else:
    498             concat_axis = _make_concat_multiindex(indexes, self.keys,

~\AppData\Local\Continuum\Anaconda3\lib\site-packages\pandas\core\reshape\concat.py in _concat_indexes(indexes)
    512 
    513 def _concat_indexes(indexes):
--> 514     return indexes[0].append(indexes[1:])
    515 
    516 

~\AppData\Local\Continuum\Anaconda3\lib\site-packages\pandas\core\indexes\multi.py in append(self, other)
   1440                 label = self._get_level_values(i)
   1441                 appended = [o._get_level_values(i) for o in other]
-> 1442                 arrays.append(label.append(appended))
   1443             return MultiIndex.from_arrays(arrays, names=self.names)
   1444 

~\AppData\Local\Continuum\Anaconda3\lib\site-packages\pandas\core\indexes\base.py in append(self, other)
   1684             # if calling index is category, don't check dtype of others
   1685             from pandas.core.indexes.category import CategoricalIndex
-> 1686             return CategoricalIndex._append_same_dtype(self, to_concat, name)
   1687 
   1688         typs = _concat.get_dtype_kinds(to_concat)

~\AppData\Local\Continuum\Anaconda3\lib\site-packages\pandas\core\indexes\category.py in _append_same_dtype(self, to_concat, name)
    624         ValueError if other is not in the categories
    625         """
--> 626         to_concat = [self._is_dtype_compat(c) for c in to_concat]
    627         codes = np.concatenate([c.codes for c in to_concat])
    628         result = self._create_from_codes(codes, name=name)

~\AppData\Local\Continuum\Anaconda3\lib\site-packages\pandas\core\indexes\category.py in <listcomp>(.0)
    624         ValueError if other is not in the categories
    625         """
--> 626         to_concat = [self._is_dtype_compat(c) for c in to_concat]
    627         codes = np.concatenate([c.codes for c in to_concat])
    628         result = self._create_from_codes(codes, name=name)

~\AppData\Local\Continuum\Anaconda3\lib\site-packages\pandas\core\indexes\category.py in _is_dtype_compat(self, other)
    186                 other = other._values
    187             if not other.is_dtype_equal(self):
--> 188                 raise TypeError("categories must match existing categories "
    189                                 "when appending")
    190         else:

TypeError: categories must match existing categories when appending
```



Discussion:
Possibly worth noting, although in this case the Categorical columns are marked as having `known` metadata, in the real case I ran across (with multiple parquet files) had unknown meta and indeed forcefully changing them here does not change the behavior.

```
In [58]: ddf['c1'].cat.known
Out[58]: True

In [60]: ddf['c1'] = ddf['c1'].cat.as_unknown()

In [61]: ddf['c2'] = ddf['c2'].cat.as_unknown()

In [63]: ddf.groupby(['c1','c2'])['v'].sum().compute()
<snip>
TypeError: categories must match existing categories when appending
```
And this is fairly obvious, but for the record, making all the categoricals known is a functional workaround for now.

```python
In [27]: ddf.categorize().groupby(['c1', 'c2'])['v'].sum().compute()
Out[27]: 
c1  c2
a   a     5
b   b     7
c   c     3
d   d     6
Name: v, dtype: int64
```
I don't personally have a solution for you on this one nor the immediate time to resolve this.  I suspect that @TomAugspurger or @jcrist might have a better understanding.  However I think that both of them are also busy for the next week or so.  

Regardless, thank you for the reproducible example.  This is perfect.  

Resolving patch:
diff --git a/dask/dataframe/methods.py b/dask/dataframe/methods.py
--- a/dask/dataframe/methods.py
+++ b/dask/dataframe/methods.py
@@ -206,6 +206,14 @@ def pivot_count(df, index, columns, values):
 # concat
 # ---------------------------------
 
+if PANDAS_VERSION < '0.20.0':
+    def _get_level_values(x, n):
+        return x.get_level_values(n)
+else:
+    def _get_level_values(x, n):
+        return x._get_level_values(n)
+
+
 def concat(dfs, axis=0, join='outer', uniform=False):
     """Concatenate, handling some edge cases:
 
@@ -233,10 +241,27 @@ def concat(dfs, axis=0, join='outer', uniform=False):
         if isinstance(dfs[0], pd.CategoricalIndex):
             return pd.CategoricalIndex(union_categoricals(dfs),
                                        name=dfs[0].name)
+        elif isinstance(dfs[0], pd.MultiIndex):
+            first, rest = dfs[0], dfs[1:]
+            if all((isinstance(o, pd.MultiIndex) and o.nlevels >= first.nlevels)
+                    for o in rest):
+                arrays = [concat([_get_level_values(i, n) for i in dfs])
+                          for n in range(first.nlevels)]
+                return pd.MultiIndex.from_arrays(arrays, names=first.names)
+
+            to_concat = (first.values, ) + tuple(k._values for k in rest)
+            new_tuples = np.concatenate(to_concat)
+            try:
+                return pd.MultiIndex.from_tuples(new_tuples, names=first.names)
+            except:
+                return pd.Index(new_tuples)
         return dfs[0].append(dfs[1:])
 
     # Handle categorical index separately
-    if isinstance(dfs[0].index, pd.CategoricalIndex):
+    dfs0_index = dfs[0].index
+    if (isinstance(dfs0_index, pd.CategoricalIndex) or
+            (isinstance(dfs0_index, pd.MultiIndex) and
+             any(isinstance(i, pd.CategoricalIndex) for i in dfs0_index.levels))):
         dfs2 = [df.reset_index(drop=True) for df in dfs]
         ind = concat([df.index for df in dfs])
     else:


Test patch:
diff --git a/dask/dataframe/tests/test_categorical.py b/dask/dataframe/tests/test_categorical.py
--- a/dask/dataframe/tests/test_categorical.py
+++ b/dask/dataframe/tests/test_categorical.py
@@ -49,6 +49,8 @@
                              y=df.y.cat.set_categories(list('abc'))))
 frames3 = [i.set_index(i.y) for i in frames]
 frames4 = [i.set_index(i.y) for i in frames2]
+frames5 = [i.set_index([i.y, i.x]) for i in frames]
+frames6 = [i.set_index([i.y, i.x]) for i in frames2]
 
 
 def test_concat_unions_categoricals():
@@ -78,6 +80,13 @@ def test_concat_unions_categoricals():
     tm.assert_series_equal(_concat([i.x for i in frames3]),
                            pd.concat([i.x for i in frames4]))
 
+    # MultiIndex with Categorical Index
+    tm.assert_index_equal(_concat([i.index for i in frames5]),
+                          pd.concat([i for i in frames6]).index)
+
+    # DataFrame, MultiIndex with CategoricalIndex
+    tm.assert_frame_equal(_concat(frames5), pd.concat(frames6))
+
 
 def test_unknown_categoricals():
     ddf = dd.DataFrame({('unknown', i): df for (i, df) in enumerate(frames)},