Repository: dask/dask

Issue:
Test failure with pandas 1.3.3
There is a new test failure on main with the latest pandas release 1.3.3. I am looking into it now. 

```python-traceback
=================================== FAILURES ===================================
___________________________ test_groupby_index_array ___________________________
[gw0] linux -- Python 3.9.7 /usr/share/miniconda3/envs/test-environment/bin/python

    def test_groupby_index_array():
        df = _compat.makeTimeDataFrame()
        ddf = dd.from_pandas(df, npartitions=2)
    
        # first select column, then group
>       assert_eq(
            df.A.groupby(df.index.month).nunique(),
            ddf.A.groupby(ddf.index.month).nunique(),
            check_names=False,
        )

dask/dataframe/tests/test_groupby.py:478: 
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 

a = 1    21
2     9
Name: A, dtype: int64
b = 2000-01-03    1
2000-01-04    1
2000-01-05    1
2000-01-06    1
2000-01-07    1
2000-01-10    1
2000-01-11    1
2000-0...1
2000-02-04    1
2000-02-07    1
2000-02-08    1
2000-02-09    1
2000-02-10    1
2000-02-11    1
Name: A, dtype: int64
check_names = False, check_dtype = True, check_divisions = True
check_index = True, kwargs = {}

    def assert_eq(
        a,
        b,
        check_names=True,
        check_dtype=True,
        check_divisions=True,
        check_index=True,
        **kwargs,
    ):
        if check_divisions:
            assert_divisions(a)
            assert_divisions(b)
            if hasattr(a, "divisions") and hasattr(b, "divisions"):
                at = type(np.asarray(a.divisions).tolist()[0])  # numpy to python
                bt = type(np.asarray(b.divisions).tolist()[0])  # scalar conversion
                assert at == bt, (at, bt)
        assert_sane_keynames(a)
        assert_sane_keynames(b)
        a = _check_dask(a, check_names=check_names, check_dtypes=check_dtype)
        b = _check_dask(b, check_names=check_names, check_dtypes=check_dtype)
        if not check_index:
            a = a.reset_index(drop=True)
            b = b.reset_index(drop=True)
        if hasattr(a, "to_pandas"):
            a = a.to_pandas()
        if hasattr(b, "to_pandas"):
            b = b.to_pandas()
        if isinstance(a, pd.DataFrame):
            a = _maybe_sort(a)
            b = _maybe_sort(b)
            tm.assert_frame_equal(a, b, check_dtype=check_dtype, **kwargs)
        elif isinstance(a, pd.Series):
            a = _maybe_sort(a)
            b = _maybe_sort(b)
>           tm.assert_series_equal(
                a, b, check_names=check_names, check_dtype=check_dtype, **kwargs
E               AssertionError: Series are different
E               
E               Series length are different
E               [left]:  2, Int64Index([1, 2], dtype='int64')
E               [right]: 30, DatetimeIndex(['2000-01-03', '2000-01-04', '2000-01-05', '2000-01-06',
E                              '2000-01-07', '2000-01-10', '2000-01-11', '2000-01-12',
E                              '2000-01-13', '2000-01-14', '2000-01-17', '2000-01-18',
E                              '2000-01-19', '2000-01-20', '2000-01-21', '2000-01-24',
E                              '2000-01-25', '2000-01-26', '2000-01-27', '2000-01-28',
E                              '2000-01-31', '2000-02-01', '2000-02-02', '2000-02-03',
E                              '2000-02-04', '2000-02-07', '2000-02-08', '2000-02-09',
E                              '2000-02-10', '2000-02-11'],
E                             dtype='datetime64[ns]', freq=None)

dask/dataframe/utils.py:559: AssertionError
```


Discussion:
I think something is wrong with `nunique` specifically. I just tried with `count` and the outcome is as expected.
It boils down to a change in the output from groupby.apply in pandas. Compare:

```python
In [1]: import pandas as pd
   ...: from dask.dataframe import _compat
   ...: 
   ...: df = _compat.makeTimeDataFrame()
   ...: g = df.groupby(df.index.month)
   ...: g.apply(lambda x: x.drop_duplicates()).index
````
pandas 1.2.5:
```python
Out[1]: 
MultiIndex([(1, '2000-01-03'),
            (1, '2000-01-04'),
            (1, '2000-01-05'),
            (1, '2000-01-06'),
            (1, '2000-01-07'),
            (1, '2000-01-10'),
            (1, '2000-01-11'),
            (1, '2000-01-12'),
            (1, '2000-01-13'),
            (1, '2000-01-14'),
            (1, '2000-01-17'),
            (1, '2000-01-18'),
            (1, '2000-01-19'),
            (1, '2000-01-20'),
            (1, '2000-01-21'),
            (1, '2000-01-24'),
            (1, '2000-01-25'),
            (1, '2000-01-26'),
            (1, '2000-01-27'),
            (1, '2000-01-28'),
            (1, '2000-01-31'),
            (2, '2000-02-01'),
            (2, '2000-02-02'),
            (2, '2000-02-03'),
            (2, '2000-02-04'),
            (2, '2000-02-07'),
            (2, '2000-02-08'),
            (2, '2000-02-09'),
            (2, '2000-02-10'),
            (2, '2000-02-11')],
           )
```

pandas 1.3.3:
```python
Out[1]: 
DatetimeIndex(['2000-01-03', '2000-01-04', '2000-01-05', '2000-01-06',
               '2000-01-07', '2000-01-10', '2000-01-11', '2000-01-12',
               '2000-01-13', '2000-01-14', '2000-01-17', '2000-01-18',
               '2000-01-19', '2000-01-20', '2000-01-21', '2000-01-24',
               '2000-01-25', '2000-01-26', '2000-01-27', '2000-01-28',
               '2000-01-31', '2000-02-01', '2000-02-02', '2000-02-03',
               '2000-02-04', '2000-02-07', '2000-02-08', '2000-02-09',
               '2000-02-10', '2000-02-11'],
              dtype='datetime64[ns]', freq='B')
```


@jorisvandenbossche do you know if this is an intentional change? From reading the groupby kwargs it looks like `group_keys` is meant to control this behavior, but that kwarg is True by default.

The behavior also differs from the behavior if you pass a string as the `by`. 

```python
In [1]: import numpy as np
   ...: import pandas as pd
   ...: 
   ...: strings = list("aaabbccccdddeee")
   ...: data = np.random.randn(len(strings))
   ...: df = pd.DataFrame(dict(strings=strings, data=data))
   ...: df.groupby("strings")[["data"]].apply(lambda x: x.drop_duplicates()).index
Out[1]: 
MultiIndex([('a',  0),
            ('a',  1),
            ('a',  2),
            ('b',  3),
            ('b',  4),
            ('c',  5),
            ('c',  6),
            ('c',  7),
            ('c',  8),
            ('d',  9),
            ('d', 10),
            ('d', 11),
            ('e', 12),
            ('e', 13),
            ('e', 14)],
           names=['strings', None])
```
I think this PR on pandas would resolve this behavior https://github.com/pandas-dev/pandas/pull/34998 
That last pandas example in https://github.com/dask/dask/issues/8137#issuecomment-919141783 doesn't seem to have changed between 1.2.5 and 1.3.3?
Right, that one always returns the groups as well as the original index. This is the one that changed: https://github.com/dask/dask/issues/8137#issuecomment-918531004
@jsignell do you know which output we actually want? If we want the "old" way, we need to trick pandas into thinking that the index has changed.

edit: whoops this example is broken.

```python
In [31]: def f(x):
    ...:     result = x.drop_duplicates()
    ...:     result.index = result.index.copy()
    ...:     return result

In [32]: g.apply(lambda x: x.drop_duplicates()).index
Out[32]:
DatetimeIndex(['2000-01-03', '2000-01-04', '2000-01-05', '2000-01-06',
               '2000-01-07', '2000-01-10', '2000-01-11', '2000-01-12',
               '2000-01-13', '2000-01-14', '2000-01-17', '2000-01-18',
               '2000-01-19', '2000-01-20', '2000-01-21', '2000-01-24',
               '2000-01-25', '2000-01-26', '2000-01-27', '2000-01-28',
               '2000-01-31', '2000-02-01', '2000-02-02', '2000-02-03',
               '2000-02-04', '2000-02-07', '2000-02-08', '2000-02-09',
               '2000-02-10', '2000-02-11'],
              dtype='datetime64[ns]', freq='B')

In [33]: g.apply(lambda x: f).index
Out[33]: Int64Index([1, 2], dtype='int64')
```

Index.copy is shallow, so the actual data shouldn't be copied, so this should be pretty cheap.

https://github.com/pandas-dev/pandas/pull/34998 is the change we want to pandas, but I don't think that's likely to land any time soon.
OK, so pure pandas reproducer (just replacing dask with pandas when calling the `makeTimeDataFrame`):

```python
import pandas as pd
df = pd._testing.makeTimeDataFrame()
df.groupby(df.index.month).apply(lambda x: x.drop_duplicates())
```

Tom's explanation is correct about the underlying issue (did the resulting index change or not?) / the discussion going on in https://github.com/pandas-dev/pandas/pull/34998. But, we still should not have changed this behaviour in a bug-fix release, though (the actual referenced PR that might change this is not merged)? Opened https://github.com/pandas-dev/pandas/issues/43568


>  If we want the "old" way, we need to trick pandas into thinking that the index has changed.

@TomAugspurger there is a small mistake in your example. In the second case, the `lambda x: f` is not actually calling the function, so that needs to be `lambda x: f(x)` (or simply `apply(f)`). 
However, when fixing that, this still gives the new behaviour of not giving a MultiIndex (so it doesn't do anything different than the pure `lambda x: x.drop_duplicates()`). 
Ah sorry about the incorrect example :/


BTW, it's a bit unfortunate this was only catched after the release. But I suppose the upstream builds have been failing too much to notice those additional failures ..
> BTW, it's a bit unfortunate this was only catched after the release. But I suppose the upstream builds have been failing too much to notice those additional failures ..

Totally. I have been trying to crack away at them but there are still outstanding failures. 
> @jsignell do you know which output we actually want? If we want the "old" way, we need to trick pandas into thinking that the index has changed.

I missed this yesterday but @TomAugspurger I think we need the old way or a newer way where only the group names are included in the index and the original index is dropped. I spent a while trying to come up with a workaround, but I couldn't quite work it out. The best I could come up with was iterating over the groups rather than using apply.
One option is to ensure that the index of the result of the apply changes by using `.reset_index()` and then setting it on the final result.

```python
In [2]: g.apply(lambda x: x.drop_duplicates().reset_index()).set_index("index", append=True).reset_index(level=1, drop=True)
Out[2]:
                     A         B         C         D
  index
1 2000-01-03  0.157718  0.208422 -0.810228 -0.231653
  2000-01-04  0.043298  0.680880 -1.932960  0.868835
  2000-01-05 -1.078813 -0.669639  0.082566 -0.271798
  2000-01-06  0.909364 -0.154112  1.129153  0.279976
  2000-01-07 -1.100145 -1.483878  0.335552  0.696921
  2000-01-10  1.127182  0.511261  1.132620  0.202625
  2000-01-11  0.473436 -0.578616 -2.167737  0.384516
  2000-01-12 -0.103394  0.940092  0.266427 -0.098527
  2000-01-13 -0.027968 -1.658127 -0.838380 -1.167990
  2000-01-14  1.255710 -0.473858  0.582221  1.610839
  2000-01-17  1.645977  0.502274  0.539010  0.510899
  2000-01-18  0.061095  0.438861 -0.268544  0.090984
  2000-01-19  0.956693 -1.353296  0.289653 -1.472087
  2000-01-20 -1.850985 -0.164340  1.250544 -1.548723
  2000-01-21 -0.439021  0.585811 -1.231420  0.155995
  2000-01-24  1.076488  0.087231  1.164339  0.108402
  2000-01-25  0.136114  0.899036  0.589867 -0.393874
  2000-01-26 -0.749156 -0.846065 -0.650573  0.654256
  2000-01-27  1.008373 -0.534091 -0.801148  1.859322
  2000-01-28 -0.797690  0.387673  1.375210 -1.814312
  2000-01-31  0.733566  0.578650 -0.844374  0.131120
2 2000-02-01  0.682151 -0.889528 -0.989753 -0.156015
  2000-02-02 -0.165678  0.166745 -1.871444 -1.085702
  2000-02-03 -0.095074  0.130935  0.376277  0.648797
  2000-02-04 -0.022004 -2.169180  0.439135  0.127369
  2000-02-07 -0.738533 -0.899933  0.315910 -0.902643
  2000-02-08 -1.053074 -0.834449 -0.982850 -1.331716
  2000-02-09 -2.162737  1.892639  0.190229 -1.234565
  2000-02-10 -0.381232 -0.131755  0.056305  0.215972
  2000-02-11  0.850109  1.379108  0.002339  0.674363
```

Or we can skip the tests for now and see if the old behavior is restored for pandas 1.3.4. This is really hard to work around.
Yeah I did skip the test for now in #8142. 

Just FYI there are a lot more of this kind of issue popping up in the upstream tests. Here's an example:

```python-traceback
__________ test_dataframe_aggregations_multilevel[nunique-<lambda>1] ___________
[gw0] linux -- Python 3.8.10 /home/julia/conda/envs/dask-upstream/bin/python

grouper = <function <lambda> at 0x7feff705c0d0>, agg_func = 'nunique'

    @pytest.mark.parametrize(
        "grouper",
        [
            lambda df: ["a"],
            lambda df: ["a", "b"],
            lambda df: df["a"],
            lambda df: [df["a"], df["b"]],
            lambda df: [df["a"] > 2, df["b"] > 1],
        ],
    )
    def test_dataframe_aggregations_multilevel(grouper, agg_func):
        def call(g, m, **kwargs):
            return getattr(g, m)(**kwargs)
    
        pdf = pd.DataFrame(
            {
                "a": [1, 2, 6, 4, 4, 6, 4, 3, 7] * 10,
                "b": [4, 2, 7, 3, 3, 1, 1, 1, 2] * 10,
                "d": [0, 1, 2, 3, 4, 5, 6, 7, 8] * 10,
                "c": [0, 1, 2, 3, 4, 5, 6, 7, 8] * 10,
            },
            columns=["c", "b", "a", "d"],
        )
    
        ddf = dd.from_pandas(pdf, npartitions=10)
    
        # covariance only works with N+1 columns
        if agg_func not in ("cov", "corr"):
>           assert_eq(
                call(pdf.groupby(grouper(pdf))["c"], agg_func),
                call(ddf.groupby(grouper(ddf))["c"], agg_func, split_every=2),
            )

dask/dataframe/tests/test_groupby.py:1085: 
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 
dask/dataframe/utils.py:544: in assert_eq
    b = _check_dask(b, check_names=check_names, check_dtypes=check_dtype)
dask/dataframe/utils.py:456: in _check_dask
    result = dsk.compute(scheduler="sync")
dask/base.py:286: in compute
    (result,) = compute(self, traverse=False, **kwargs)
dask/base.py:568: in compute
    results = schedule(dsk, keys, **kwargs)
dask/local.py:560: in get_sync
    return get_async(
dask/local.py:503: in get_async
    for key, res_info, failed in queue_get(queue).result():
../conda/envs/dask-upstream/lib/python3.8/concurrent/futures/_base.py:437: in result
    return self.__get_result()
../conda/envs/dask-upstream/lib/python3.8/concurrent/futures/_base.py:389: in __get_result
    raise self._exception
dask/local.py:545: in submit
    fut.set_result(fn(*args, **kwargs))
dask/local.py:237: in batch_execute_tasks
    return [execute_task(*a) for a in it]
dask/local.py:237: in <listcomp>
    return [execute_task(*a) for a in it]
dask/local.py:228: in execute_task
    result = pack_exception(e, dumps)
dask/local.py:223: in execute_task
    result = _execute_task(task, data)
dask/core.py:121: in _execute_task
    return func(*(_execute_task(a, cache) for a in args))
dask/utils.py:35: in apply
    return func(*args, **kwargs)
dask/dataframe/groupby.py:511: in _nunique_df_chunk
    [grouped.index.get_level_values(level=level) for level in levels]
dask/dataframe/groupby.py:511: in <listcomp>
    [grouped.index.get_level_values(level=level) for level in levels]
../conda/envs/dask-upstream/lib/python3.8/site-packages/pandas/core/indexes/base.py:1869: in _get_level_values
    self._validate_index_level(level)
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 

self = Int64Index([0, 1, 2, 3, 4, 5, 6, 7, 8], dtype='int64'), level = 1

    @final
    def _validate_index_level(self, level) -> None:
        """
        Validate index level.
    
        For single-level Index getting level number is a no-op, but some
        verification must be done like in MultiIndex.
    
        """
        if isinstance(level, int):
            if level < 0 and level != -1:
                raise IndexError(
                    "Too many levels: Index has only 1 level, "
                    f"{level} is not a valid level number"
                )
            elif level > 0:
>               raise IndexError(
                    f"Too many levels: Index has only 1 level, not {level + 1}"
E                   IndexError: Too many levels: Index has only 1 level, not 2

../conda/envs/dask-upstream/lib/python3.8/site-packages/pandas/core/indexes/base.py:1788: IndexError
```
ref https://github.com/dask/dask/issues/8036
> One option is to ensure that the index of the result of the apply changes by using .reset_index() and then setting it on the final result.

That is a nice workaround. I hadn't thought of that one :slightly_smiling_face: 

Resolving patch:
diff --git a/dask/dataframe/groupby.py b/dask/dataframe/groupby.py
--- a/dask/dataframe/groupby.py
+++ b/dask/dataframe/groupby.py
@@ -391,6 +391,11 @@ def _mul_cols(df, cols):
     for i, j in it.combinations_with_replacement(cols, 2):
         col = "%s%s" % (i, j)
         _df[col] = df[i] * df[j]
+
+    # Fix index in a groupby().apply() context
+    # https://github.com/dask/dask/issues/8137
+    # https://github.com/pandas-dev/pandas/issues/43568
+    _df.index = [0] * len(_df)
     return _df
 
 
@@ -429,8 +434,8 @@ def _cov_chunk(df, *index):
     g = _groupby_raise_unaligned(df, by=index)
     x = g.sum()
 
-    level = len(index)
-    mul = g.apply(_mul_cols, cols=cols).reset_index(level=level, drop=True)
+    mul = g.apply(_mul_cols, cols=cols).reset_index(level=-1, drop=True)
+
     n = g[x.columns].count().rename(columns=lambda c: "{}-count".format(c))
     return (x, mul, n, col_mapping)
 
@@ -495,23 +500,23 @@ def _cov_agg(_t, levels, ddof, std=False, sort=False):
 ###############################################################
 # nunique
 ###############################################################
+def _drop_duplicates_reindex(df):
+    # Fix index in a groupby().apply() context
+    # https://github.com/dask/dask/issues/8137
+    # https://github.com/pandas-dev/pandas/issues/43568
+    result = df.drop_duplicates()
+    result.index = [0] * len(result)
+    return result
 
 
 def _nunique_df_chunk(df, *index, **kwargs):
-    levels = kwargs.pop("levels")
     name = kwargs.pop("name")
 
     g = _groupby_raise_unaligned(df, by=index)
     if len(df) > 0:
-        grouped = g[[name]].apply(M.drop_duplicates)
-        # we set the index here to force a possibly duplicate index
-        # for our reduce step
-        if isinstance(levels, list):
-            grouped.index = pd.MultiIndex.from_arrays(
-                [grouped.index.get_level_values(level=level) for level in levels]
-            )
-        else:
-            grouped.index = grouped.index.get_level_values(level=levels)
+        grouped = (
+            g[[name]].apply(_drop_duplicates_reindex).reset_index(level=-1, drop=True)
+        )
     else:
         # Manually create empty version, since groupby-apply for empty frame
         # results in df with no columns
@@ -521,24 +526,12 @@ def _nunique_df_chunk(df, *index, **kwargs):
     return grouped
 
 
-def _drop_duplicates_rename(df):
-    # Avoid duplicate index labels in a groupby().apply() context
-    # https://github.com/dask/dask/issues/3039
-    # https://github.com/pandas-dev/pandas/pull/18882
-    names = [None] * df.index.nlevels
-    return df.drop_duplicates().rename_axis(names, copy=False)
-
-
 def _nunique_df_combine(df, levels, sort=False):
-    result = df.groupby(level=levels, sort=sort).apply(_drop_duplicates_rename)
-
-    if isinstance(levels, list):
-        result.index = pd.MultiIndex.from_arrays(
-            [result.index.get_level_values(level=level) for level in levels]
-        )
-    else:
-        result.index = result.index.get_level_values(level=levels)
-
+    result = (
+        df.groupby(level=levels, sort=sort)
+        .apply(_drop_duplicates_reindex)
+        .reset_index(level=-1, drop=True)
+    )
     return result
 
 


Test patch:
diff --git a/dask/dataframe/tests/test_groupby.py b/dask/dataframe/tests/test_groupby.py
--- a/dask/dataframe/tests/test_groupby.py
+++ b/dask/dataframe/tests/test_groupby.py
@@ -9,7 +9,7 @@
 import dask
 import dask.dataframe as dd
 from dask.dataframe import _compat
-from dask.dataframe._compat import PANDAS_GT_110, PANDAS_GT_133, tm
+from dask.dataframe._compat import PANDAS_GT_110, tm
 from dask.dataframe.utils import assert_dask_graph, assert_eq, assert_max_deps
 from dask.utils import M
 
@@ -470,7 +470,6 @@ def test_series_groupby_errors():
         ss.groupby("x")  # dask should raise the same error
 
 
-@pytest.mark.skipif(PANDAS_GT_133, reason="https://github.com/dask/dask/issues/8137")
 def test_groupby_index_array():
     df = _compat.makeTimeDataFrame()
     ddf = dd.from_pandas(df, npartitions=2)