Repository: dask/dask

Issue:
Cannot reset index of dask.dataframe when underyling pandas Dataframes have multiIndex
There appears to be an issue when resetting the index of a `dask.dataframe` after performing a `dd.groupby` when the resulting dask dataframe contains a multi index:

```
import pandas as pd
import numpy as np
import dask.dataframe as dd

df = pd.DataFrame(np.random.rand(2,3), columns=['A', 'B', 'C'])
ddf = dd.from_pandas(df, npartitions=1)
grouped = ddf.groupby(['A', 'B']).agg({'C': 'sum'})
grouped = grouped.reset_index()
grouped.compute()
```

Traceback:
```
Traceback (most recent call last):
  File "gist.py", line 9, in <module>
    grouped.compute()
  File "/Users/rwest/anaconda/envs/test_dask/lib/python2.7/site-packages/dask/base.py", line 95, in compute
    (result,) = compute(self, traverse=False, **kwargs)
  File "/Users/rwest/anaconda/envs/test_dask/lib/python2.7/site-packages/dask/base.py", line 202, in compute
    results = get(dsk, keys, **kwargs)
  File "/Users/rwest/anaconda/envs/test_dask/lib/python2.7/site-packages/dask/threaded.py", line 76, in get
    **kwargs)
  File "/Users/rwest/anaconda/envs/test_dask/lib/python2.7/site-packages/dask/async.py", line 500, in get_async
    raise(remote_exception(res, tb))
dask.async.ValueError: Length mismatch: Expected axis has 3 elements, new values have 2 elements

Traceback
---------
  File "/Users/rwest/anaconda/envs/test_dask/lib/python2.7/site-packages/dask/async.py", line 266, in execute_task
    result = _execute_task(task, data)
  File "/Users/rwest/anaconda/envs/test_dask/lib/python2.7/site-packages/dask/async.py", line 247, in _execute_task
    return func(*args2)
  File "/Users/rwest/anaconda/envs/test_dask/lib/python2.7/site-packages/dask/dataframe/core.py", line 3111, in apply_and_enforce
    return _rename(c, df)
  File "/Users/rwest/anaconda/envs/test_dask/lib/python2.7/site-packages/dask/dataframe/core.py", line 3148, in _rename
    df.columns = columns
  File "/Users/rwest/anaconda/envs/test_dask/lib/python2.7/site-packages/pandas/core/generic.py", line 2757, in __setattr__
    return object.__setattr__(self, name, value)
  File "pandas/src/properties.pyx", line 65, in pandas.lib.AxisProperty.__set__ (pandas/lib.c:46249)
  File "/Users/rwest/anaconda/envs/test_dask/lib/python2.7/site-packages/pandas/core/generic.py", line 448, in _set_axis
    self._data.set_axis(axis, labels)
  File "/Users/rwest/anaconda/envs/test_dask/lib/python2.7/site-packages/pandas/core/internals.py", line 2802, in set_axis
    (old_len, new_len))
```

Env: (`conda create --name test_dask python=2.7.13 dask=0.14.1 pandas=0.19.2 numpy=1.12.1`)
```
backports                 1.0                      py27_0  
backports_abc             0.5                      py27_0  
bokeh                     0.12.4                   py27_0  
chest                     0.2.3                    py27_0  
cloudpickle               0.2.2                    py27_0  
dask                      0.14.1                   py27_0  
futures                   3.0.5                    py27_0  
heapdict                  1.0.0                    py27_1  
jinja2                    2.9.6                    py27_0  
locket                    0.2.0                    py27_1  
markupsafe                0.23                     py27_2  
mkl                       2017.0.1                      0  
numpy                     1.12.1                   py27_0  
openssl                   1.0.2k                        1  
pandas                    0.19.2              np112py27_1  
partd                     0.3.7                    py27_0  
pip                       9.0.1                    py27_1  
python                    2.7.13                        0  
python-dateutil           2.6.0                    py27_0  
pytz                      2017.2                   py27_0  
pyyaml                    3.12                     py27_0  
readline                  6.2                           2  
requests                  2.13.0                   py27_0  
setuptools                27.2.0                   py27_0  
singledispatch            3.4.0.3                  py27_0  
six                       1.10.0                   py27_0  
sqlite                    3.13.0                        0  
ssl_match_hostname        3.4.0.2                  py27_1  
tk                        8.5.18                        0  
toolz                     0.8.2                    py27_0  
tornado                   4.5.1                    py27_0  
wheel                     0.29.0                   py27_0  
yaml                      0.1.6                         0  
zlib                      1.2.8                         3  
```






Discussion:
Thanks for the bug report and excellent test case. I've found the issue, fix coming soon.

Resolving patch:
diff --git a/dask/dataframe/groupby.py b/dask/dataframe/groupby.py
--- a/dask/dataframe/groupby.py
+++ b/dask/dataframe/groupby.py
@@ -866,14 +866,6 @@ def aggregate(self, arg, split_every, split_out=1):
         else:
             levels = 0
 
-        # apply the transformations to determine the meta object
-        meta_groupby = pd.Series([], dtype=bool, index=self.obj._meta.index)
-        meta_stage1 = _groupby_apply_funcs(self.obj._meta, funcs=chunk_funcs,
-                                           by=meta_groupby)
-        meta_stage2 = _groupby_apply_funcs(meta_stage1, funcs=aggregate_funcs,
-                                           level=0)
-        meta = _agg_finalize(meta_stage2, finalizers)
-
         if not isinstance(self.index, list):
             chunk_args = [self.obj, self.index]
 
@@ -887,11 +879,11 @@ def aggregate(self, arg, split_every, split_out=1):
                   aggregate_kwargs=dict(funcs=aggregate_funcs, level=levels),
                   combine=_groupby_apply_funcs,
                   combine_kwargs=dict(funcs=aggregate_funcs, level=levels),
-                  meta=meta, token='aggregate', split_every=split_every,
+                  token='aggregate', split_every=split_every,
                   split_out=split_out, split_out_setup=split_out_on_index)
 
-        return map_partitions(_agg_finalize, obj, meta=meta,
-                              token='aggregate-finalize', funcs=finalizers)
+        return map_partitions(_agg_finalize, obj, token='aggregate-finalize',
+                              funcs=finalizers)
 
     @insert_meta_param_description(pad=12)
     def apply(self, func, meta=no_default, columns=no_default):
@@ -1043,10 +1035,6 @@ def __init__(self, df, by=None, slice=None):
 
     def nunique(self, split_every=None, split_out=1):
         name = self._meta.obj.name
-        meta = pd.Series([], dtype='int64',
-                         index=pd.Index([], dtype=self._meta.obj.dtype),
-                         name=name)
-
         levels = _determine_levels(self.index)
 
         if isinstance(self.obj, DataFrame):
@@ -1059,7 +1047,7 @@ def nunique(self, split_every=None, split_out=1):
                    chunk=chunk,
                    aggregate=_nunique_df_aggregate,
                    combine=_nunique_df_combine,
-                   meta=meta, token='series-groupby-nunique',
+                   token='series-groupby-nunique',
                    chunk_kwargs={'levels': levels, 'name': name},
                    aggregate_kwargs={'levels': levels, 'name': name},
                    combine_kwargs={'levels': levels},
diff --git a/dask/dataframe/reshape.py b/dask/dataframe/reshape.py
--- a/dask/dataframe/reshape.py
+++ b/dask/dataframe/reshape.py
@@ -132,6 +132,7 @@ def pivot_table(df, index=None, columns=None,
     # the result must have CategoricalIndex columns
     new_columns = pd.CategoricalIndex(df[columns].cat.categories, name=columns)
     meta = pd.DataFrame(columns=new_columns, dtype=np.float64)
+    meta.index.name = index
 
     kwargs = {'index': index, 'columns': columns, 'values': values}
 
diff --git a/dask/dataframe/utils.py b/dask/dataframe/utils.py
--- a/dask/dataframe/utils.py
+++ b/dask/dataframe/utils.py
@@ -432,41 +432,42 @@ def meta_nonempty(x):
 ###############################################################
 
 
-def _check_dask(dsk, check_names=True, check_dtypes=True):
+def _check_dask(dsk, check_names=True, check_dtypes=True, result=None):
     import dask.dataframe as dd
     if hasattr(dsk, 'dask'):
-        result = dsk.compute(get=get_sync)
+        if result is None:
+            result = dsk.compute(get=get_sync)
         if isinstance(dsk, dd.Index):
             assert isinstance(result, pd.Index), type(result)
-            if check_names:
-                assert dsk.name == result.name
-            # cache
             assert isinstance(dsk._meta, pd.Index), type(dsk._meta)
             if check_names:
+                assert dsk.name == result.name
                 assert dsk._meta.name == result.name
+                if isinstance(result, pd.MultiIndex):
+                    assert result.names == dsk._meta.names
             if check_dtypes:
                 assert_dask_dtypes(dsk, result)
         elif isinstance(dsk, dd.Series):
             assert isinstance(result, pd.Series), type(result)
-            if check_names:
-                assert dsk.name == result.name, (dsk.name, result.name)
-            # cache
             assert isinstance(dsk._meta, pd.Series), type(dsk._meta)
             if check_names:
+                assert dsk.name == result.name, (dsk.name, result.name)
                 assert dsk._meta.name == result.name
             if check_dtypes:
                 assert_dask_dtypes(dsk, result)
+            _check_dask(dsk.index, check_names=check_names,
+                        check_dtypes=check_dtypes, result=result.index)
         elif isinstance(dsk, dd.DataFrame):
             assert isinstance(result, pd.DataFrame), type(result)
             assert isinstance(dsk.columns, pd.Index), type(dsk.columns)
-            if check_names:
-                tm.assert_index_equal(dsk.columns, result.columns)
-            # cache
             assert isinstance(dsk._meta, pd.DataFrame), type(dsk._meta)
             if check_names:
+                tm.assert_index_equal(dsk.columns, result.columns)
                 tm.assert_index_equal(dsk._meta.columns, result.columns)
             if check_dtypes:
                 assert_dask_dtypes(dsk, result)
+            _check_dask(dsk.index, check_names=check_names,
+                        check_dtypes=check_dtypes, result=result.index)
         elif isinstance(dsk, dd.core.Scalar):
             assert (np.isscalar(result) or
                     isinstance(result, (pd.Timestamp, pd.Timedelta)))


Test patch:
diff --git a/dask/dataframe/tests/test_groupby.py b/dask/dataframe/tests/test_groupby.py
--- a/dask/dataframe/tests/test_groupby.py
+++ b/dask/dataframe/tests/test_groupby.py
@@ -258,11 +258,8 @@ def test_series_groupby_propagates_names():
     df = pd.DataFrame({'x': [1, 2, 3], 'y': [4, 5, 6]})
     ddf = dd.from_pandas(df, 2)
     func = lambda df: df['y'].sum()
-
-    result = ddf.groupby('x').apply(func, meta=('y', 'i8'))
-
+    result = ddf.groupby('x').apply(func)
     expected = df.groupby('x').apply(func)
-    expected.name = 'y'
     assert_eq(result, expected)