Repository: dask/dask

Issue:
DataFrame groupby apply returns unexpected duplicated MultiIndex
```
import pandas as pd
import dask.dataframe as dd
df = pd.DataFrame([[1, 2], [2, 3], [3, 4]], columns=['a', 'b'])
# df = df.astype({'b': 'category'})
df = dd.from_pandas(df, npartitions=1).set_index('a')
df = df.groupby('a').apply(lambda d: d.copy())
df.compute()
```

returns a dataframe with a MultiIndex where both levels are just copies of the original index:

```
     b
a a   
1 1  2
2 2  3
3 3  4
```

Providing the right meta object does not change the result. Interestingly, if I convert column 'b' to 'category' dtype (uncomment the line `# df = df.astype({'b': 'category'})`), I get the right result:

```
   b
a   
1  2
2  3
3  4
```


Discussion:
Pandas has some inconsistencies here when the input  is (maybe) mutate in side an apply.

```python
In [17]: df.groupby('a').apply(lambda d: d)
Out[17]:
   a  b
0  1  2
1  2  3
2  3  4

In [18]: df.groupby('a').apply(lambda d: d.copy())
Out[18]:
     a  b
a
1 0  1  2
2 1  2  3
3 2  3  4
```

I'd recommend searching on the pandas issue tracker to see if this is the expected behavior, and maybe make a PR fixing it if not.
Ah okay. It appears to be expected behavior in Pandas, a result of Pandas `.groupby(group_keys=True)` being the default. If `group_keys=False` the result is what I would want:

```
>>> df.groupby('a', group_keys=False).apply(lambda d: d.copy())
   a  b
0  1  2
1  2  3
2  3  4
```

So the fix here would be to expose `group_keys` in dask.
It's not clear to me that the difference between the two pandas outputs
(mutate vs. not) is intentional.

On Thu, Mar 14, 2019 at 3:10 PM Brian Chu <notifications@github.com> wrote:

> Ah okay. It appears to be expected behavior in Pandas, a result of Pandas
> .groupby(group_keys=True) being the default. If group_keys=False the
> result is what we would want:
>
> >>> df.groupby('a', group_keys=False).apply(lambda d: d.copy())
>    a  b
> 0  1  2
> 1  2  3
> 2  3  4
>
> So the fix here would be to expose group_keys in dask.
>
> —
> You are receiving this because you commented.
> Reply to this email directly, view it on GitHub
> <https://github.com/dask/dask/issues/4592#issuecomment-473041010>, or mute
> the thread
> <https://github.com/notifications/unsubscribe-auth/ABQHIsDgg6PlF_L73rVhwukRHKIiHi3wks5vWqzGgaJpZM4bzBTx>
> .
>

It does appear to be a pandas bug (https://github.com/pandas-dev/pandas/issues/22546), where the "correct" Pandas behavior is the behavior exhibited by `df.groupby('a').apply(lambda d: d.copy())` (the grouping index gets added).

So I suppose this is more of a feature request for supporting `.groupby(group_keys)`

Resolving patch:
diff --git a/dask/dataframe/groupby.py b/dask/dataframe/groupby.py
--- a/dask/dataframe/groupby.py
+++ b/dask/dataframe/groupby.py
@@ -144,7 +144,8 @@ def _groupby_raise_unaligned(df, **kwargs):
 def _groupby_slice_apply(df, grouper, key, func, *args, **kwargs):
     # No need to use raise if unaligned here - this is only called after
     # shuffling, which makes everything aligned already
-    g = df.groupby(grouper)
+    group_keys = kwargs.pop('group_keys', True)
+    g = df.groupby(grouper, group_keys=group_keys)
     if key:
         g = g[key]
     return g.apply(func, *args, **kwargs)
@@ -762,10 +763,13 @@ class _GroupBy(object):
         The key for grouping
     slice: str, list
         The slice keys applied to GroupBy result
+    group_keys: bool
+        Passed to pandas.DataFrame.groupby()
     """
-    def __init__(self, df, by=None, slice=None):
+    def __init__(self, df, by=None, slice=None, group_keys=True):
 
         assert isinstance(df, (DataFrame, Series))
+        self.group_keys = group_keys
         self.obj = df
         # grouping key passed via groupby method
         self.index = _normalize_index(df, by)
@@ -796,7 +800,7 @@ def __init__(self, df, by=None, slice=None):
         else:
             index_meta = self.index
 
-        self._meta = self.obj._meta.groupby(index_meta)
+        self._meta = self.obj._meta.groupby(index_meta, group_keys=group_keys)
 
     @property
     def _meta_nonempty(self):
@@ -814,7 +818,7 @@ def _meta_nonempty(self):
         else:
             index_meta = self.index
 
-        grouped = sample.groupby(index_meta)
+        grouped = sample.groupby(index_meta, group_keys=self.group_keys)
         return _maybe_slice(grouped, self._slice)
 
     def _aca_agg(self, token, func, aggfunc=None, split_every=None,
@@ -1191,7 +1195,7 @@ def apply(self, func, *args, **kwargs):
         # Perform embarrassingly parallel groupby-apply
         kwargs['meta'] = meta
         df5 = map_partitions(_groupby_slice_apply, df4, index2,
-                             self._slice, func, token=funcname(func), *args,
+                             self._slice, func, token=funcname(func), *args, group_keys=self.group_keys,
                              **kwargs)
 
         return df5
@@ -1237,7 +1241,7 @@ class SeriesGroupBy(_GroupBy):
 
     _token_prefix = 'series-groupby-'
 
-    def __init__(self, df, by=None, slice=None):
+    def __init__(self, df, by=None, slice=None, **kwargs):
         # for any non series object, raise pandas-compat error message
 
         if isinstance(df, Series):
@@ -1255,7 +1259,7 @@ def __init__(self, df, by=None, slice=None):
                 # raise error from pandas, if applicable
                 df._meta.groupby(by)
 
-        super(SeriesGroupBy, self).__init__(df, by=by, slice=slice)
+        super(SeriesGroupBy, self).__init__(df, by=by, slice=slice, **kwargs)
 
     def nunique(self, split_every=None, split_out=1):
         name = self._meta.obj.name


Test patch:
diff --git a/dask/dataframe/tests/test_groupby.py b/dask/dataframe/tests/test_groupby.py
--- a/dask/dataframe/tests/test_groupby.py
+++ b/dask/dataframe/tests/test_groupby.py
@@ -1153,8 +1153,6 @@ def test_groupby_not_supported():
         ddf.groupby('A', as_index=False)
     with pytest.raises(TypeError):
         ddf.groupby('A', sort=False)
-    with pytest.raises(TypeError):
-        ddf.groupby('A', group_keys=False)
     with pytest.raises(TypeError):
         ddf.groupby('A', squeeze=True)
 
@@ -1597,3 +1595,19 @@ def test_with_min_count(min_count):
                   ddf.groupby('group').sum(min_count=min_count))
         assert_eq(df.groupby('group').prod(min_count=min_count),
                   ddf.groupby('group').prod(min_count=min_count))
+
+
+def test_groupby_group_keys():
+    df = pd.DataFrame({
+        'a': [1, 2, 2, 3],
+        'b': [2, 3, 4, 5],
+    })
+    ddf = dd.from_pandas(df, npartitions=2).set_index('a')
+    pdf = df.set_index('a')
+
+    func = lambda g: g.copy()
+    expected = pdf.groupby('a').apply(func)
+    assert_eq(expected, ddf.groupby('a').apply(func, meta=expected))
+
+    expected = pdf.groupby('a', group_keys=False).apply(func)
+    assert_eq(expected, ddf.groupby('a', group_keys=False).apply(func, meta=expected))