Repository: blaze/blaze

Issue:
postgres nunique on table fails
In this example, `es` is a postgres table with the following dshape:

``` python
In [15]: es.f69185227fdf48b98f08fdf6382a974b.dshape
Out[15]: 
dshape("""var * {
  asof_date: datetime,
  open: ?float64,
  high: ?float64,
  low: ?float64,
  close: ?float64,
  volume: ?float64,
  adjusted_close: ?float64
  }""")
```

Calling `nunique` on the table itself raises a programming error from psycopg2.

``` python
In [11]: es = bz.Data('postgresql://localhost/bz')
In [12]: es.f69185227fdf48b98f08fdf6382a974b.nunique()
...
ProgrammingError: (psycopg2.ProgrammingError) function nunique() does not exis
LINE 1: SELECT nunique() AS f69185227fdf48b98f08fdf6382a974b_nunique...
               ^
HINT:  No function matches the given name and argument types. You might need to add explicit type casts.
 [SQL: 'SELECT nunique() AS f69185227fdf48b98f08fdf6382a974b_nunique \nFROM f69185227fdf48b98f08fdf6382a974b']
```

I imagine it should issue `nunique(*)` or something; however, I am not sure yet where to make this change.



Discussion:
the sql expression would be

``` sql
select count(*) from (select distinct * from table) as tmp;
```

this can be done in blaze with

``` python
db.table.distinct().count()
```

Okay; should this raise a `NotImplementedError` then or do you think that this should continue to bubble up the `ProgrammingError`?

I actually think this should work.

I've got an implementation in the pipeline

Awesome


Resolving patch:
diff --git a/blaze/compute/pandas.py b/blaze/compute/pandas.py
--- a/blaze/compute/pandas.py
+++ b/blaze/compute/pandas.py
@@ -35,7 +35,7 @@
                     Map, Apply, Merge, std, var, Like, Slice, summary,
                     ElemWise, DateTime, Millisecond, Expr, Symbol, IsIn,
                     UTCFromTimestamp, nelements, DateTimeTruncate, count,
-                    UnaryStringFunction)
+                    UnaryStringFunction, nunique)
 from ..expr import UnaryOp, BinOp
 from ..expr import symbol, common_subexpression
 from .core import compute, compute_up, base
@@ -160,6 +160,11 @@ def compute_up(t, df, **kwargs):
     return df.drop_duplicates().reset_index(drop=True)
 
 
+@dispatch(nunique, DataFrame)
+def compute_up(expr, data, **kwargs):
+    return compute_up(expr._child.distinct().count(), data, **kwargs)
+
+
 string_func_names = {
     'strlen': 'len',
 }
diff --git a/blaze/compute/sql.py b/blaze/compute/sql.py
--- a/blaze/compute/sql.py
+++ b/blaze/compute/sql.py
@@ -404,7 +404,12 @@ def compute_up(t, s, **kwargs):
     return sa.func.count(s.distinct())
 
 
-@dispatch(Distinct, sa.Table)
+@dispatch(nunique, Selectable)
+def compute_up(expr, data, **kwargs):
+    return select(data).distinct().alias(next(aliases)).count()
+
+
+@dispatch(Distinct, Selectable)
 def compute_up(t, s, **kwargs):
     return select(s).distinct()
 


Test patch:
diff --git a/blaze/compute/tests/test_mongo_compute.py b/blaze/compute/tests/test_mongo_compute.py
--- a/blaze/compute/tests/test_mongo_compute.py
+++ b/blaze/compute/tests/test_mongo_compute.py
@@ -217,6 +217,10 @@ def test_distinct(bank):
     assert set(compute(t.name.distinct(), bank)) == set(['Alice', 'Bob'])
 
 
+def test_nunique_collection(bank):
+    assert compute(t.nunique(), bank) == len(bank_raw)
+
+
 def test_sort(bank):
     assert compute(t.amount.sort('amount'), bank) == \
         [100, 100, 200, 200, 300]
diff --git a/blaze/compute/tests/test_numpy_compute.py b/blaze/compute/tests/test_numpy_compute.py
--- a/blaze/compute/tests/test_numpy_compute.py
+++ b/blaze/compute/tests/test_numpy_compute.py
@@ -429,3 +429,12 @@ def test_isin(keys):
     result = compute(s.x.isin(keys), b)
     expected = np.in1d(b['x'], keys)
     np.testing.assert_array_equal(result, expected)
+
+
+def test_nunique_recarray():
+    b = np.array([('a', 1), ('b', 2), ('c', 3), ('a', 4), ('c', 5), ('b', 6),
+                  ('a', 1), ('b', 2)],
+                 dtype=[('x', 'S1'), ('y', 'i4')])
+    s = symbol('s', discover(b))
+    expr = s.nunique()
+    assert compute(expr, b) == len(np.unique(b))
diff --git a/blaze/compute/tests/test_pandas_compute.py b/blaze/compute/tests/test_pandas_compute.py
--- a/blaze/compute/tests/test_pandas_compute.py
+++ b/blaze/compute/tests/test_pandas_compute.py
@@ -662,3 +662,9 @@ def test_isin(keys):
     result = compute(expr, df)
     expected = df.loc[df.id.isin(keys)]
     tm.assert_frame_equal(result, expected)
+
+
+def test_nunique_table():
+    expr = t.nunique()
+    result = compute(expr, df)
+    assert result == len(df.drop_duplicates())
diff --git a/blaze/compute/tests/test_sparksql.py b/blaze/compute/tests/test_sparksql.py
--- a/blaze/compute/tests/test_sparksql.py
+++ b/blaze/compute/tests/test_sparksql.py
@@ -362,3 +362,8 @@ def test_isin(ctx, db, keys):
     expected = odo(compute(expr, {db: {'t': df}}), set)
     assert (set(map(frozenset, odo(result, list))) ==
             set(map(frozenset, odo(expected, list))))
+
+
+def test_nunique_spark_dataframe(ctx, db):
+    assert (odo(compute(db.t.nunique(), ctx), int) ==
+            ctx.table('t').distinct().count())
diff --git a/blaze/compute/tests/test_sql_compute.py b/blaze/compute/tests/test_sql_compute.py
--- a/blaze/compute/tests/test_sql_compute.py
+++ b/blaze/compute/tests/test_sql_compute.py
@@ -334,10 +334,18 @@ def test_nunique():
     assert 'amount' in result.lower()
 
 
+def test_nunique_table():
+    result = normalize(str(computefull(t.nunique(), s)))
+    expected = normalize("""SELECT count(alias.id) AS tbl_row_count
+FROM (SELECT DISTINCT accounts.name AS name, accounts.amount AS amount, accounts.id AS id
+FROM accounts) as alias""")
+    assert result == expected
+
+
 @xfail(reason="Fails because SQLAlchemy doesn't seem to know binary reductions")
 def test_binary_reductions():
     assert str(compute(any(t['amount'] > 150), s)) == \
-            str(sa.sql.functions.any(s.c.amount > 150))
+        str(sa.sql.functions.any(s.c.amount > 150))
 
 
 def test_by():