Repository: dask/dask

Issue:
Feature: implement SeriesGroupBy.{value_counts, unique}
I'm trying to break some custom aggregation functions down into chunks / agg / finalize as in the [docs](http://docs.dask.org/en/latest/dataframe-groupby.html?highlight=set_index%20disk#aggregate), but I'm stumbling over the fact that `SeriesGroupBy` in dask (as of v2.1.0) cannot perform `value_counts()`.

This seems like an operation that could be easily distributed (each partition does the value counts separately, and then all that's necessary is adding the chunk-results together).

Admittedly, this can cause a lot of traffic if the Series contains only unique values (i.e. every value_count is 1), but that's just the nature of the beast.

Similarly, `SeriesGroupby.unique` could be implemented. This is orders of magnitude slower in pandas than the (pandas-)`value_counts`, but is conceptually even simpler than `value_counts` - rather than count the appearances per value, we just need to keep track which values appeared at all.

Since `pd.SeriesGroupBy.unique` performs so much worse than `pd.SeriesGroupBy.value_counts`, it would even suffice to implement `dask.SeriesGroupBy.unique` through `dask.SeriesGroupBy.value_counts` (essentially by discarding the count).


Discussion:
I agree that this would be a useful addition. Would you be interested in submitting a PR?
@jcrist 
I'll try to have a look. Could you point me in the direction where this should be implemented, and maybe after what method I could model it?
Sure. The relevant code is this class: https://github.com/dask/dask/blob/2c7ac8808ead66d4c217948f9693d7b4039e66ae/dask/dataframe/groupby.py#L1557

You'll be adding a new reduction method. Reductions are usually created using the `_aca_agg` private method - "aca" stands for "apply-concatenate-apply", which describes the parallelism model (one function is applied to all the chunks, the results are concatenated, then another function is applied to the results). The trick here will be writing these two methods. See the other reductions for some examples.

Resolving patch:
diff --git a/dask/dataframe/groupby.py b/dask/dataframe/groupby.py
--- a/dask/dataframe/groupby.py
+++ b/dask/dataframe/groupby.py
@@ -1784,3 +1784,37 @@ def aggregate(self, arg, split_every=None, split_out=1):
     @derived_from(pd.core.groupby.SeriesGroupBy)
     def agg(self, arg, split_every=None, split_out=1):
         return self.aggregate(arg, split_every=split_every, split_out=split_out)
+
+    @derived_from(pd.core.groupby.SeriesGroupBy)
+    def value_counts(self, split_every=None, split_out=1):
+        return self._aca_agg(
+            token="value_counts",
+            func=M.value_counts,
+            aggfunc=_value_counts_aggregate,
+            split_every=split_every,
+            split_out=split_out,
+        )
+
+    @derived_from(pd.core.groupby.SeriesGroupBy)
+    def unique(self, split_every=None, split_out=1):
+        name = self._meta.obj.name
+        return self._aca_agg(
+            token="unique",
+            func=M.unique,
+            aggfunc=_unique_aggregate,
+            aggregate_kwargs={"name": name},
+            split_every=split_every,
+            split_out=split_out,
+        )
+
+
+def _unique_aggregate(series_gb, name=None):
+    ret = pd.Series({k: v.explode().unique() for k, v in series_gb}, name=name)
+    ret.index.names = series_gb.obj.index.names
+    return ret
+
+
+def _value_counts_aggregate(series_gb):
+    to_concat = {k: v.sum(level=1) for k, v in series_gb}
+    names = list(series_gb.obj.index.names)
+    return pd.Series(pd.concat(to_concat, names=names))
diff --git a/dask/utils.py b/dask/utils.py
--- a/dask/utils.py
+++ b/dask/utils.py
@@ -549,7 +549,7 @@ def extra_titles(doc):
     titles = {
         i: lines[i].strip()
         for i in range(len(lines) - 1)
-        if lines[i + 1] and all(c == "-" for c in lines[i + 1].strip())
+        if lines[i + 1].strip() and all(c == "-" for c in lines[i + 1].strip())
     }
 
     seen = set()
@@ -616,6 +616,11 @@ def _derived_from(cls, method, ua_args=[], extra=""):
     # do not use wraps here, as it hides keyword arguments displayed
     # in the doc
     original_method = getattr(cls, method.__name__)
+
+    if isinstance(original_method, property):
+        # some things like SeriesGroupBy.unique are generated.
+        original_method = original_method.fget
+
     doc = original_method.__doc__
     if doc is None:
         doc = ""


Test patch:
diff --git a/dask/dataframe/tests/test_groupby.py b/dask/dataframe/tests/test_groupby.py
--- a/dask/dataframe/tests/test_groupby.py
+++ b/dask/dataframe/tests/test_groupby.py
@@ -1,5 +1,6 @@
 import collections
 import warnings
+from packaging import version
 
 import numpy as np
 import pandas as pd
@@ -2094,6 +2095,36 @@ def test_series_groupby_idxmax_skipna(skipna):
     assert_eq(result_pd, result_dd)
 
 
+@pytest.mark.skipif(
+    version.parse(pd.__version__) < version.parse("0.25.0"),
+    reason="'explode' is not implemented",
+)
+def test_groupby_unique():
+    rng = np.random.RandomState(42)
+    df = pd.DataFrame(
+        {"foo": rng.randint(3, size=100), "bar": rng.randint(10, size=100)}
+    )
+    ddf = dd.from_pandas(df, npartitions=10)
+
+    pd_gb = df.groupby("foo")["bar"].unique()
+    dd_gb = ddf.groupby("foo")["bar"].unique()
+
+    # Use explode because each DataFrame row is a list; equality fails
+    assert_eq(dd_gb.explode(), pd_gb.explode())
+
+
+def test_groupby_value_counts():
+    rng = np.random.RandomState(42)
+    df = pd.DataFrame(
+        {"foo": rng.randint(3, size=100), "bar": rng.randint(4, size=100)}
+    )
+    ddf = dd.from_pandas(df, npartitions=2)
+
+    pd_gb = df.groupby("foo")["bar"].value_counts()
+    dd_gb = ddf.groupby("foo")["bar"].value_counts()
+    assert_eq(dd_gb, pd_gb)
+
+
 @pytest.mark.parametrize(
     "transformation", [lambda x: x.sum(), np.sum, "sum", pd.Series.rank]
 )