Repository: dask/dask

Issue:
value_counts groupby multiple keys, wrong index
```python
import dask

ddf = dask.datasets.timeseries()
# simple column for value_counts
ddf['bool_tocount'] = False
ddf['bool_tocount'] = ddf['x'] < 0

pdf = ddf.head(10000)
```
expected result (pandas):
```python
pdf.groupby(['id', 'name'])['bool_tocount'].value_counts().head()
```

```bash
id   name    bool_tocount
884  Quinn   False           1
885  Bob     False           1
890  Oliver  True            1
891  Sarah   True            1
894  Dan     True            1
Name: bool_tocount, dtype: int64
```

wrong result (dask): wrong meaning index `bool_tocount` values are names and not True/False.
```python
ddf.groupby(['id', 'name'])['bool_tocount'].value_counts().head()
```

```bash
id   name    bool_tocount
877  Ray     Ray             2
880  Quinn   Quinn           2
882  Ingrid  Ingrid          3
883  Bob     Bob             2
     Oliver  Oliver          1
Name: bool_tocount, dtype: int64
```
I think it's the `level=1` in here: https://github.com/dask/dask/blob/ceec30358545599a6b978224cecb4e01b14dd517/dask/dataframe/groupby.py#L1858

Don't know the best way to fix it in a general manner. Ideas?

Thanks



Discussion:

Resolving patch:
diff --git a/dask/dataframe/groupby.py b/dask/dataframe/groupby.py
--- a/dask/dataframe/groupby.py
+++ b/dask/dataframe/groupby.py
@@ -11,6 +11,7 @@
 
 from dask import config
 from dask.base import is_dask_collection, tokenize
+from dask.core import flatten
 from dask.dataframe._compat import (
     PANDAS_GT_140,
     PANDAS_GT_150,
@@ -2823,16 +2824,19 @@ def _unique_aggregate(series_gb, name=None):
 
 
 def _value_counts(x, **kwargs):
-    if len(x):
-        return M.value_counts(x, **kwargs)
-    else:
+    if not x.groups or all(
+        pd.isna(key) for key in flatten(x.groups.keys(), container=tuple)
+    ):
         return pd.Series(dtype=int)
+    else:
+        return x.value_counts(**kwargs)
 
 
 def _value_counts_aggregate(series_gb):
-    to_concat = {k: v.groupby(level=1).sum() for k, v in series_gb}
-    names = list(series_gb.obj.index.names)
-    return pd.Series(pd.concat(to_concat, names=names))
+    return pd.concat(
+        {k: v.groupby(level=-1).sum() for k, v in series_gb},
+        names=series_gb.obj.index.names,
+    )
 
 
 def _tail_chunk(series_gb, **kwargs):


Test patch:
diff --git a/dask/dataframe/tests/test_groupby.py b/dask/dataframe/tests/test_groupby.py
--- a/dask/dataframe/tests/test_groupby.py
+++ b/dask/dataframe/tests/test_groupby.py
@@ -2388,15 +2388,20 @@ def test_groupby_unique():
     assert_eq(dd_gb.explode(), pd_gb.explode())
 
 
-def test_groupby_value_counts():
+@pytest.mark.parametrize("by", ["foo", ["foo", "bar"]])
+def test_groupby_value_counts(by):
     rng = np.random.RandomState(42)
     df = pd.DataFrame(
-        {"foo": rng.randint(3, size=100), "bar": rng.randint(4, size=100)}
+        {
+            "foo": rng.randint(3, size=100),
+            "bar": rng.randint(4, size=100),
+            "baz": rng.randint(5, size=100),
+        }
     )
     ddf = dd.from_pandas(df, npartitions=2)
 
-    pd_gb = df.groupby("foo")["bar"].value_counts()
-    dd_gb = ddf.groupby("foo")["bar"].value_counts()
+    pd_gb = df.groupby(by).baz.value_counts()
+    dd_gb = ddf.groupby(by).baz.value_counts()
     assert_eq(dd_gb, pd_gb)
 
 
@@ -3028,26 +3033,27 @@ def test_series_named_agg(shuffle, agg):
     assert_eq(expected, actual)
 
 
-def test_empty_partitions_with_value_counts():
+@pytest.mark.parametrize("by", ["A", ["A", "B"]])
+def test_empty_partitions_with_value_counts(by):
     # https://github.com/dask/dask/issues/7065
     df = pd.DataFrame(
         data=[
-            ["a1", "b1"],
-            ["a1", None],
-            ["a1", "b1"],
-            [None, None],
-            [None, None],
-            [None, None],
-            ["a3", "b3"],
-            ["a3", "b3"],
-            ["a5", "b5"],
+            ["a1", "b1", True],
+            ["a1", None, False],
+            ["a1", "b1", True],
+            [None, None, None],
+            [None, None, None],
+            [None, None, None],
+            ["a3", "b3", True],
+            ["a3", "b3", False],
+            ["a5", "b5", True],
         ],
-        columns=["A", "B"],
+        columns=["A", "B", "C"],
     )
 
-    expected = df.groupby("A")["B"].value_counts()
+    expected = df.groupby(by).C.value_counts()
     ddf = dd.from_pandas(df, npartitions=3)
-    actual = ddf.groupby("A")["B"].value_counts()
+    actual = ddf.groupby(by).C.value_counts()
     assert_eq(expected, actual)