Repository: dask/dask

Issue:
⚠️ Upstream CI failed ⚠️
[Workflow Run URL](https://github.com/dask/dask/actions/runs/2235329821)


Discussion:
Rats I was hoping this was just a transient error, but looks like we have another segfault from pandas. See https://github.com/dask/dask/issues/8768 for more comments. 

@jorisvandenbossche the segfault is coming from `test_apply_or_transform_shuffle_multilevel`
Unfortunately, we are still having the segfault on `test_apply_or_transform_shuffle_multilevel`
Did some investigation, and filed an upstream issue in pandas: https://github.com/pandas-dev/pandas/issues/46566
The upstream fix has been merged, we should be able to verify it in our CI once a new nightly lands

Resolving patch:
diff --git a/dask/dataframe/core.py b/dask/dataframe/core.py
--- a/dask/dataframe/core.py
+++ b/dask/dataframe/core.py
@@ -81,6 +81,8 @@
 
 no_default = "__no_default__"
 
+GROUP_KEYS_DEFAULT = None if PANDAS_GT_150 else True
+
 pd.set_option("compute.use_numexpr", False)
 
 
@@ -3615,7 +3617,13 @@ def _validate_axis(cls, axis=0):
 
     @derived_from(pd.Series)
     def groupby(
-        self, by=None, group_keys=True, sort=None, observed=None, dropna=None, **kwargs
+        self,
+        by=None,
+        group_keys=GROUP_KEYS_DEFAULT,
+        sort=None,
+        observed=None,
+        dropna=None,
+        **kwargs,
     ):
         from dask.dataframe.groupby import SeriesGroupBy
 
@@ -4748,7 +4756,13 @@ def nsmallest(self, n=5, columns=None, split_every=None):
 
     @derived_from(pd.DataFrame)
     def groupby(
-        self, by=None, group_keys=True, sort=None, observed=None, dropna=None, **kwargs
+        self,
+        by=None,
+        group_keys=GROUP_KEYS_DEFAULT,
+        sort=None,
+        observed=None,
+        dropna=None,
+        **kwargs,
     ):
         from dask.dataframe.groupby import DataFrameGroupBy
 
diff --git a/dask/dataframe/groupby.py b/dask/dataframe/groupby.py
--- a/dask/dataframe/groupby.py
+++ b/dask/dataframe/groupby.py
@@ -9,6 +9,7 @@
 
 from dask.base import tokenize
 from dask.dataframe.core import (
+    GROUP_KEYS_DEFAULT,
     DataFrame,
     Series,
     _extract_meta,
@@ -158,7 +159,15 @@ def _groupby_raise_unaligned(df, **kwargs):
 
 
 def _groupby_slice_apply(
-    df, grouper, key, func, *args, group_keys=True, dropna=None, observed=None, **kwargs
+    df,
+    grouper,
+    key,
+    func,
+    *args,
+    group_keys=GROUP_KEYS_DEFAULT,
+    dropna=None,
+    observed=None,
+    **kwargs,
 ):
     # No need to use raise if unaligned here - this is only called after
     # shuffling, which makes everything aligned already
@@ -171,7 +180,15 @@ def _groupby_slice_apply(
 
 
 def _groupby_slice_transform(
-    df, grouper, key, func, *args, group_keys=True, dropna=None, observed=None, **kwargs
+    df,
+    grouper,
+    key,
+    func,
+    *args,
+    group_keys=GROUP_KEYS_DEFAULT,
+    dropna=None,
+    observed=None,
+    **kwargs,
 ):
     # No need to use raise if unaligned here - this is only called after
     # shuffling, which makes everything aligned already
@@ -189,7 +206,14 @@ def _groupby_slice_transform(
 
 
 def _groupby_slice_shift(
-    df, grouper, key, shuffled, group_keys=True, dropna=None, observed=None, **kwargs
+    df,
+    grouper,
+    key,
+    shuffled,
+    group_keys=GROUP_KEYS_DEFAULT,
+    dropna=None,
+    observed=None,
+    **kwargs,
 ):
     # No need to use raise if unaligned here - this is only called after
     # shuffling, which makes everything aligned already
@@ -1035,7 +1059,7 @@ class _GroupBy:
         The key for grouping
     slice: str, list
         The slice keys applied to GroupBy result
-    group_keys: bool
+    group_keys: bool | None
         Passed to pandas.DataFrame.groupby()
     dropna: bool
         Whether to drop null values from groupby index
@@ -1053,7 +1077,7 @@ def __init__(
         df,
         by=None,
         slice=None,
-        group_keys=True,
+        group_keys=GROUP_KEYS_DEFAULT,
         dropna=None,
         sort=None,
         observed=None,


Test patch:
diff --git a/dask/dataframe/tests/test_dataframe.py b/dask/dataframe/tests/test_dataframe.py
--- a/dask/dataframe/tests/test_dataframe.py
+++ b/dask/dataframe/tests/test_dataframe.py
@@ -1412,6 +1412,10 @@ def test_empty_quantile(method):
     assert_eq(result, exp)
 
 
+# TODO: un-filter once https://github.com/dask/dask/issues/8960 is resolved.
+@pytest.mark.filterwarnings(
+    "ignore:In future versions of pandas, numeric_only will be set to False:FutureWarning"
+)
 @pytest.mark.parametrize(
     "method,expected",
     [
diff --git a/dask/dataframe/tests/test_groupby.py b/dask/dataframe/tests/test_groupby.py
--- a/dask/dataframe/tests/test_groupby.py
+++ b/dask/dataframe/tests/test_groupby.py
@@ -14,6 +14,10 @@
 from dask.dataframe.utils import assert_dask_graph, assert_eq, assert_max_deps
 from dask.utils import M
 
+CHECK_FREQ = {}
+if dd._compat.PANDAS_GT_110:
+    CHECK_FREQ["check_freq"] = False
+
 AGG_FUNCS = [
     "sum",
     "mean",
@@ -1873,7 +1877,8 @@ def test_with_min_count(min_count):
         )
 
 
-def test_groupby_group_keys():
+@pytest.mark.parametrize("group_keys", [True, False, None])
+def test_groupby_group_keys(group_keys):
     df = pd.DataFrame({"a": [1, 2, 2, 3], "b": [2, 3, 4, 5]})
     ddf = dd.from_pandas(df, npartitions=2).set_index("a")
     pdf = df.set_index("a")
@@ -1882,8 +1887,10 @@ def test_groupby_group_keys():
     expected = pdf.groupby("a").apply(func)
     assert_eq(expected, ddf.groupby("a").apply(func, meta=expected))
 
-    expected = pdf.groupby("a", group_keys=False).apply(func)
-    assert_eq(expected, ddf.groupby("a", group_keys=False).apply(func, meta=expected))
+    expected = pdf.groupby("a", group_keys=group_keys).apply(func)
+    assert_eq(
+        expected, ddf.groupby("a", group_keys=group_keys).apply(func, meta=expected)
+    )
 
 
 @pytest.mark.parametrize(
@@ -2174,8 +2181,13 @@ def test_groupby_shift_with_freq():
 
     # just pass the pandas result as meta for convenience
     df_result = pdf.groupby(pdf.index).shift(periods=-2, freq="D")
+    # Groupby/shift on the index should avoid shuffle and let the `freq` pass
+    # unmodified, but that is currently broken: https://github.com/dask/dask/issues/8959
+    # TODO: remove check_freq condition once fixed.
     assert_eq(
-        df_result, ddf.groupby(ddf.index).shift(periods=-2, freq="D", meta=df_result)
+        df_result,
+        ddf.groupby(ddf.index).shift(periods=-2, freq="D", meta=df_result),
+        **CHECK_FREQ,
     )
     df_result = pdf.groupby("b").shift(periods=-2, freq="D")
     assert_eq(df_result, ddf.groupby("b").shift(periods=-2, freq="D", meta=df_result))
@@ -2291,7 +2303,21 @@ def test_groupby_dropna_pandas(dropna):
 @pytest.mark.gpu
 @pytest.mark.parametrize("dropna", [False, True, None])
 @pytest.mark.parametrize("by", ["a", "c", "d", ["a", "b"], ["a", "c"], ["a", "d"]])
-def test_groupby_dropna_cudf(dropna, by):
+@pytest.mark.parametrize(
+    "group_keys",
+    [
+        True,
+        pytest.param(
+            False,
+            marks=pytest.mark.xfail(reason="cudf hasn't updated group_keys default"),
+        ),
+        pytest.param(
+            None,
+            marks=pytest.mark.xfail(reason="cudf hasn't updated group_keys default"),
+        ),
+    ],
+)
+def test_groupby_dropna_cudf(dropna, by, group_keys):
 
     # NOTE: This test requires cudf/dask_cudf, and will
     # be skipped by non-GPU CI
@@ -2311,11 +2337,11 @@ def test_groupby_dropna_cudf(dropna, by):
     ddf = dask_cudf.from_cudf(df, npartitions=3)
 
     if dropna is None:
-        dask_result = ddf.groupby(by).e.sum()
-        cudf_result = df.groupby(by).e.sum()
+        dask_result = ddf.groupby(by, group_keys=group_keys).e.sum()
+        cudf_result = df.groupby(by, group_keys=group_keys).e.sum()
     else:
-        dask_result = ddf.groupby(by, dropna=dropna).e.sum()
-        cudf_result = df.groupby(by, dropna=dropna).e.sum()
+        dask_result = ddf.groupby(by, dropna=dropna, group_keys=group_keys).e.sum()
+        cudf_result = df.groupby(by, dropna=dropna, group_keys=group_keys).e.sum()
     if by in ["c", "d"]:
         # Lose string/category index name in cudf...
         dask_result = dask_result.compute()
@@ -2570,6 +2596,8 @@ def test_groupby_with_pd_grouper():
         ddf.groupby(["key1", pd.Grouper(key="key2")])
 
 
+# TODO: Remove filter once https://github.com/pandas-dev/pandas/issues/46814 is resolved
+@pytest.mark.filterwarnings("ignore:Invalid value encountered:RuntimeWarning")
 @pytest.mark.parametrize("operation", ["head", "tail"])
 def test_groupby_empty_partitions_with_rows_operation(operation):