Repository: dask/dask

Issue:
nunique
We lack an `nunique` operation.  There are two possible approaches as I see it
1.  Spoof `nunique` by doing `drop_duplicates` and then count.
2.  Compute approximate `nunique` with `hyperloglog`

I suspect that in our common case `drop_duplicates` is not a bottleneck, so I would do that first (it's simpler) until someone complains about speed

This could also be done for groupbys.



Discussion:

Resolving patch:
diff --git a/dask/dataframe/core.py b/dask/dataframe/core.py
--- a/dask/dataframe/core.py
+++ b/dask/dataframe/core.py
@@ -1,3 +1,5 @@
+from __future__ import division
+
 from itertools import count
 from math import ceil, sqrt
 from functools import wraps
@@ -378,16 +380,23 @@ def __rxor__(self, other):
 
     def sum(self):
         return reduction(self, pd.Series.sum, np.sum)
+
     def max(self):
         return reduction(self, pd.Series.max, np.max)
+
     def min(self):
         return reduction(self, pd.Series.min, np.min)
+
     def count(self):
         return reduction(self, pd.Series.count, np.sum)
 
+    def nunique(self):
+        return self.drop_duplicates().count()
+
     def mean(self):
         def chunk(ser):
             return (ser.sum(), ser.count())
+
         def agg(seq):
             sums, counts = list(zip(*seq))
             return 1.0 * sum(sums) / sum(counts)
@@ -396,6 +405,7 @@ def agg(seq):
     def var(self, ddof=1):
         def chunk(ser):
             return (ser.sum(), (ser**2).sum(), ser.count())
+
         def agg(seq):
             x, x2, n = list(zip(*seq))
             x = float(sum(x))
@@ -748,12 +758,26 @@ def agg(df):
             g = df.groupby(level=0)
             x = g.agg({(self.key, 'sum'): 'sum',
                        (self.key, 'count'): 'sum'})
-            result = 1.0 * x[self.key]['sum'] / x[self.key]['count']
+            result = x[self.key]['sum'] / x[self.key]['count']
             result.name = self.key
             return result
         return aca([self.frame, self.index],
                    chunk=chunk, aggregate=agg, columns=[self.key])
 
+    def nunique(self):
+        def chunk(df, index):
+            # we call set_index here to force a possibly duplicate index
+            # for our reduce step
+            return (df.groupby(index)
+                      .apply(pd.DataFrame.drop_duplicates, subset=self.key)
+                      .set_index(index))
+
+        def agg(df):
+            return df.groupby(level=0)[self.key].nunique()
+
+        return aca([self.frame, self.index],
+                   chunk=chunk, aggregate=agg, columns=[self.key])
+
 
 def apply_concat_apply(args, chunk=None, aggregate=None, columns=None):
     """ Apply a function to blocks, the concat, then apply again


Test patch:
diff --git a/dask/dataframe/tests/test_dataframe.py b/dask/dataframe/tests/test_dataframe.py
--- a/dask/dataframe/tests/test_dataframe.py
+++ b/dask/dataframe/tests/test_dataframe.py
@@ -7,7 +7,7 @@
 import dask
 from dask.utils import raises
 import dask.dataframe as dd
-from dask.dataframe.core import get, concat, pd_split
+from dask.dataframe.core import get, concat
 
 
 def eq(a, b):
@@ -396,3 +396,29 @@ def test_random_partitions():
     assert isinstance(b, dd.DataFrame)
 
     assert len(a.compute()) + len(b.compute()) == len(full)
+
+
+def test_series_nunique():
+    ps = pd.Series(list('aaabbccccdddeee'), name='a')
+    s = dd.from_pandas(ps, npartitions=3)
+    assert s.nunique().compute() == ps.nunique()
+
+
+def test_dataframe_groupby_nunique():
+    strings = list('aaabbccccdddeee')
+    data = np.random.randn(len(strings))
+    ps = pd.DataFrame(dict(strings=strings, data=data))
+    s = dd.from_pandas(ps, npartitions=3)
+    expected = ps.groupby('strings')['data'].nunique()
+    result = s.groupby('strings')['data'].nunique().compute()
+    tm.assert_series_equal(result, expected)
+
+
+def test_dataframe_groupby_nunique_across_group_same_value():
+    strings = list('aaabbccccdddeee')
+    data = list(map(int, '123111223323412'))
+    ps = pd.DataFrame(dict(strings=strings, data=data))
+    s = dd.from_pandas(ps, npartitions=3)
+    expected = ps.groupby('strings')['data'].nunique()
+    result = s.groupby('strings')['data'].nunique().compute()
+    tm.assert_series_equal(result, expected)