Repository: blaze/blaze

Issue:
Split-apply-combine with summary on Mongo fails
``` Python
In [1]: from blaze import *
In [2]: import pymongo

In [3]: db = pymongo.MongoClient().db
In [4]: t = Table(db)
In [5]: t = Table(db.mycollection)
In [6]: t
Out[6]: 
   amount  id     name
0     100   1    Alice
1    -200   2      Bob
2     300   3  Charlie
3     400   4   Dennis
4    -500   5    Edith

In [7]: by(t.name, total=t.amount.sum())
Out[7]: <repr(<blaze.expr.table.By at 0x7f397f358140>) failed: AttributeError: 'sum' object has no attribute 'columns'>
```



Discussion:
This works without the keyword argument (straight reduction).  I just failed to implement Summary in Mongo


Resolving patch:
diff --git a/blaze/compute/mongo.py b/blaze/compute/mongo.py
--- a/blaze/compute/mongo.py
+++ b/blaze/compute/mongo.py
@@ -56,7 +56,7 @@
 from ..expr import (var, Label, std, Sort, count, nunique, Selection, mean,
                     Reduction, Head, ReLabel, Apply, Distinct, RowWise, By,
                     TableSymbol, Projection, sum, min, max, TableExpr,
-                    Gt, Lt, Ge, Le, Eq, Ne, ScalarSymbol, And, Or)
+                    Gt, Lt, Ge, Le, Eq, Ne, ScalarSymbol, And, Or, Summary)
 from ..expr.core import Expr
 
 from ..dispatch import dispatch
@@ -91,7 +91,6 @@ def __init__(self, coll, query):
     def append(self, clause):
         return MongoQuery(self.coll, self.query + (clause,))
 
-
     def info(self):
         return self.coll, self.query
 
@@ -133,7 +132,7 @@ def compute_one(t, q, **kwargs):
     if not isinstance(t.grouper, Projection):
         raise ValueError("Complex By operations not supported on MongoDB.\n"
                 "Must be of the form `by(t[columns], t[column].reduction()`")
-    name = t.apply.dshape[0].names[0]
+    names = t.apply.dshape[0].names
     return MongoQuery(q.coll, q.query +
     ({
         '$group': toolz.merge(
@@ -143,7 +142,7 @@ def compute_one(t, q, **kwargs):
      },
      {
          '$project': toolz.merge(dict((col, '$_id.'+col) for col in t.grouper.columns),
-                                 {name: '$'+name})
+                                 dict((name, '$' + name) for name in names))
      }))
 
 
@@ -155,7 +154,7 @@ def compute_one(t, q, **kwargs):
                               {'_id': 0})}))
 
 
-
+@dispatch(Reduction)
 def group_apply(expr):
     """
     Dictionary corresponding to apply part of split-apply-combine operation
@@ -166,7 +165,7 @@ def group_apply(expr):
     """
     assert isinstance(expr.dshape[0], Record)
     key = expr.dshape[0].names[0]
-    col = '$'+expr.child.columns[0]
+    col = '$' + expr.child.columns[0]
     if isinstance(expr, count):
         return {key: {'$sum': 1}}
     if isinstance(expr, sum):
@@ -178,7 +177,22 @@ def group_apply(expr):
     if isinstance(expr, mean):
         return {key: {'$avg': col}}
     raise NotImplementedError("Reduction %s not yet supported in MongoDB"
-            % type(expr).__name__)
+                              % type(expr).__name__)
+
+
+reductions = {mean: 'avg', count: 'sum'}
+
+
+@dispatch(Summary)
+def group_apply(expr):
+    # TODO: implement columns variable more generally when ColumnWise works
+    reducs = expr.values
+    columns = [c.child.column for c in reducs]
+    values = zip(expr.names, reducs, columns)
+    key_getter = lambda v: '$%s' % reductions.get(type(v), type(v).__name__)
+    query = dict((k, {key_getter(v): int(isinstance(v, count)) or '$%s' % z})
+                 for k, v, z in values)
+    return query
 
 
 @dispatch(count, MongoQuery)
@@ -192,7 +206,8 @@ def compute_one(t, q, **kwargs):
     name = t.dshape[0].names[0]
     reduction = {sum: '$sum', min: '$min', max: '$max', mean: '$avg'}[type(t)]
     column = '$' + t.child.columns[0]
-    return q.append({'$group': {'_id': {}, name: {reduction: column}}})
+    arg = {'$group': {'_id': {}, name: {reduction: column}}}
+    return q.append(arg)
 
 
 @dispatch(Sort, MongoQuery)
@@ -229,8 +244,9 @@ def post_compute(e, q, d):
 
     http://docs.mongodb.org/manual/core/aggregation-pipeline/
     """
-    q = q.append({'$project': toolz.merge({'_id': 0}, # remove mongo identifier
-                                      dict((col, 1) for col in e.columns))})
+    d = {'$project': toolz.merge({'_id': 0},  # remove mongo identifier
+                                 dict((col, 1) for col in e.columns))}
+    q = q.append(d)
     dicts = q.coll.aggregate(list(q.query))['result']
 
     if e.iscolumn:


Test patch:
diff --git a/blaze/compute/tests/test_mongo_compute.py b/blaze/compute/tests/test_mongo_compute.py
--- a/blaze/compute/tests/test_mongo_compute.py
+++ b/blaze/compute/tests/test_mongo_compute.py
@@ -3,13 +3,7 @@
 import pytest
 pymongo = pytest.importorskip('pymongo')
 
-try:
-    pymongo.MongoClient()
-except pymongo.errors.ConnectionFailure:
-    pytest.importorskip('fhskjfdskfhsf')
-
 from datetime import datetime
-from contextlib import contextmanager
 from toolz import pluck
 
 from blaze import into, compute, compute_one
@@ -18,140 +12,192 @@
 from blaze.expr import TableSymbol, by
 from blaze.compatibility import xfail
 
-conn = pymongo.MongoClient()
-db = conn.test_db
 
+@pytest.fixture(scope='module')
+def conn():
+    try:
+        return pymongo.MongoClient()
+    except pymongo.errors.ConnectionFailure:
+        pytest.skip('No mongo server running')
+
+
+@pytest.fixture(scope='module')
+def db(conn):
+    return conn.test_db
+
+
+@pytest.fixture
+def bank_raw():
+    data = [{'name': 'Alice', 'amount': 100},
+            {'name': 'Alice', 'amount': 200},
+            {'name': 'Bob', 'amount': 100},
+            {'name': 'Bob', 'amount': 200},
+            {'name': 'Bob', 'amount': 300}]
+    return data
 
-@contextmanager
-def collection(data=[]):
+
+@pytest.yield_fixture
+def bank(db, bank_raw):
     coll = db.tmp_collection
-    if data:
-        coll = into(coll, data)
+    coll = into(coll, bank_raw)
+    yield coll
+    coll.drop()
+
+
+@pytest.yield_fixture
+def points(db):
+    data = [{'x': 1, 'y': 10, 'z': 100},
+            {'x': 2, 'y': 20, 'z': 200},
+            {'x': 3, 'y': 30, 'z': 300},
+            {'x': 4, 'y': 40, 'z': 400}]
+    coll = db.tmp_collection
+    coll = into(coll, data)
+    yield coll
+    coll.drop()
 
-    try:
-        yield coll
-    finally:
-        coll.drop()
 
+@pytest.yield_fixture
+def events(db):
+    data = [{'time': datetime(2012, 1, 1, 12, 00, 00), 'x': 1},
+            {'time': datetime(2012, 1, 2, 12, 00, 00), 'x': 2},
+            {'time': datetime(2012, 1, 3, 12, 00, 00), 'x': 3}]
+    coll = db.tmp_collection
+    coll = into(coll, data)
+    yield coll
+    coll.drop()
+
+
+@pytest.fixture
+def t():
+    return TableSymbol('t', '{name: string, amount: int}')
 
-bank = [{'name': 'Alice', 'amount': 100},
-        {'name': 'Alice', 'amount': 200},
-        {'name': 'Bob', 'amount': 100},
-        {'name': 'Bob', 'amount': 200},
-        {'name': 'Bob', 'amount': 300}]
 
-points = [{'x': 1, 'y': 10, 'z': 100},
-          {'x': 2, 'y': 20, 'z': 200},
-          {'x': 3, 'y': 30, 'z': 300},
-          {'x': 4, 'y': 40, 'z': 400}]
+@pytest.fixture
+def p():
+    return TableSymbol('p', '{x: int, y: int, z: int}')
 
-events = [{'time': datetime(2012, 1, 1, 12, 00, 00), 'x': 1},
-          {'time': datetime(2012, 1, 2, 12, 00, 00), 'x': 2},
-          {'time': datetime(2012, 1, 3, 12, 00, 00), 'x': 3}]
 
-t = TableSymbol('t', '{name: string, amount: int}')
+@pytest.fixture
+def e():
+    return TableSymbol('e', '{time: datetime, x: int}')
 
-p = TableSymbol('p', '{x: int, y: int, z: int}')
 
-e = TableSymbol('e', '{time: datetime, x: int}')
+@pytest.fixture
+def q():
+    return MongoQuery('fake', [])
 
-q = MongoQuery('fake', [])
 
-def test_tablesymbol_one():
-    with collection(bank) as coll:
-        assert compute_one(t, coll) == MongoQuery(coll, ())
+def test_tablesymbol_one(t, bank):
+    assert compute_one(t, bank) == MongoQuery(bank, ())
 
-def test_tablesymbol():
-    with collection(bank) as coll:
-        assert compute(t, coll) == list(pluck(['name', 'amount'], bank))
 
-def test_projection_one():
+def test_tablesymbol(t, bank, bank_raw):
+    assert compute(t, bank) == list(pluck(['name', 'amount'], bank_raw))
+
+
+def test_projection_one(t, q):
     assert compute_one(t[['name']], q).query == ({'$project': {'name': 1}},)
 
-def test_head_one():
+
+def test_head_one(t, q):
     assert compute_one(t.head(5), q).query == ({'$limit': 5},)
 
-def test_head():
-    with collection(bank) as coll:
-        assert len(compute(t.head(2), coll)) == 2
-
-def test_projection():
-    with collection(bank) as coll:
-        assert set(compute(t.name, coll)) == set(['Alice', 'Bob'])
-        assert set(compute(t[['name']], coll)) == set([('Alice',), ('Bob',)])
-
-
-def test_selection():
-    with collection(bank) as coll:
-        assert set(compute(t[t.name=='Alice'], coll)) == set([('Alice', 100),
-                                                              ('Alice', 200)])
-        assert set(compute(t['Alice'==t.name], coll)) == set([('Alice', 100),
-                                                              ('Alice', 200)])
-        assert set(compute(t[t.amount > 200], coll)) == set([('Bob', 300)])
-        assert set(compute(t[t.amount >= 200], coll)) == set([('Bob', 300),
-                                                              ('Bob', 200),
-                                                              ('Alice', 200)])
-        assert set(compute(t[t.name!='Alice'].name, coll)) == set(['Bob'])
-        assert set(compute(t[(t.name=='Alice') & (t.amount > 150)], coll)) == \
-                set([('Alice', 200)])
-        assert set(compute(t[(t.name=='Alice') | (t.amount > 250)], coll)) == \
-                set([('Alice', 200),
-                     ('Alice', 100),
-                     ('Bob', 300)])
-
-
-@xfail()
-def test_columnwise():
-    with collection(points) as coll:
-        assert set(compute(p.x + p.y, coll)) == set([11, 22, 33])
-
-
-def test_by_one():
+
+def test_head(t, bank):
+    assert len(compute(t.head(2), bank)) == 2
+
+
+def test_projection(t, bank):
+    assert set(compute(t.name, bank)) == set(['Alice', 'Bob'])
+    assert set(compute(t[['name']], bank)) == set([('Alice',), ('Bob',)])
+
+
+def test_selection(t, bank):
+    assert set(compute(t[t.name=='Alice'], bank)) == set([('Alice', 100),
+                                                            ('Alice', 200)])
+    assert set(compute(t['Alice'==t.name], bank)) == set([('Alice', 100),
+                                                            ('Alice', 200)])
+    assert set(compute(t[t.amount > 200], bank)) == set([('Bob', 300)])
+    assert set(compute(t[t.amount >= 200], bank)) == set([('Bob', 300),
+                                                          ('Bob', 200),
+                                                          ('Alice', 200)])
+    assert set(compute(t[t.name!='Alice'].name, bank)) == set(['Bob'])
+    assert set(compute(t[(t.name=='Alice') & (t.amount > 150)], bank)) == \
+            set([('Alice', 200)])
+    assert set(compute(t[(t.name=='Alice') | (t.amount > 250)], bank)) == \
+            set([('Alice', 200),
+                    ('Alice', 100),
+                    ('Bob', 300)])
+
+
+@xfail(raises=NotImplementedError,
+       reason='ColumnWise not implemented for MongoDB')
+def test_columnwise(p, points):
+    assert set(compute(p.x + p.y, points)) == set([11, 22, 33])
+
+
+def test_by_one(t, q):
     assert compute_one(by(t.name, t.amount.sum()), q).query == \
             ({'$group': {'_id': {'name': '$name'},
                          'amount_sum': {'$sum': '$amount'}}},
              {'$project': {'amount_sum': '$amount_sum', 'name': '$_id.name'}})
 
-def test_by():
-    with collection(bank) as coll:
-        assert set(compute(by(t.name, t.amount.sum()), coll)) == \
-                set([('Alice', 300), ('Bob', 600)])
-        assert set(compute(by(t.name, t.amount.min()), coll)) == \
-                set([('Alice', 100), ('Bob', 100)])
-        assert set(compute(by(t.name, t.amount.max()), coll)) == \
-                set([('Alice', 200), ('Bob', 300)])
-        assert set(compute(by(t.name, t.name.count()), coll)) == \
-                set([('Alice', 2), ('Bob', 3)])
-
-
-def test_reductions():
-    with collection(bank) as coll:
-        assert compute(t.amount.min(), coll) == 100
-        assert compute(t.amount.max(), coll) == 300
-        assert compute(t.amount.sum(), coll) == 900
-
-
-def test_distinct():
-    with collection(bank) as coll:
-        assert set(compute(t.name.distinct(), coll)) == set(['Alice', 'Bob'])
-
-
-def test_sort():
-    with collection(bank) as coll:
-        assert compute(t.amount.sort('amount'), coll) == \
-                [100, 100, 200, 200, 300]
-        assert compute(t.amount.sort('amount', ascending=False), coll) == \
-                [300, 200, 200, 100, 100]
-
-def test_by_multi_column():
-    with collection(bank) as coll:
-        assert set(compute(by(t[['name', 'amount']], t.count()), coll)) == \
-                set([(d['name'], d['amount'], 1) for d in bank])
-
-
-def test_datetime_handling():
-    with collection(events) as coll:
-        assert set(compute(e[e.time >= datetime(2012, 1, 2, 12, 0, 0)].x,
-                           coll)) == set([2, 3])
-        assert set(compute(e[e.time >= "2012-01-02"].x,
-                           coll)) == set([2, 3])
+
+def test_by(t, bank):
+    assert set(compute(by(t.name, t.amount.sum()), bank)) == \
+            set([('Alice', 300), ('Bob', 600)])
+    assert set(compute(by(t.name, t.amount.min()), bank)) == \
+            set([('Alice', 100), ('Bob', 100)])
+    assert set(compute(by(t.name, t.amount.max()), bank)) == \
+            set([('Alice', 200), ('Bob', 300)])
+    assert set(compute(by(t.name, t.name.count()), bank)) == \
+            set([('Alice', 2), ('Bob', 3)])
+
+
+def test_reductions(t, bank):
+    assert compute(t.amount.min(), bank) == 100
+    assert compute(t.amount.max(), bank) == 300
+    assert compute(t.amount.sum(), bank) == 900
+
+
+def test_distinct(t, bank):
+    assert set(compute(t.name.distinct(), bank)) == set(['Alice', 'Bob'])
+
+
+def test_sort(t, bank):
+    assert compute(t.amount.sort('amount'), bank) == \
+            [100, 100, 200, 200, 300]
+    assert compute(t.amount.sort('amount', ascending=False), bank) == \
+            [300, 200, 200, 100, 100]
+
+
+def test_by_multi_column(t, bank, bank_raw):
+    assert set(compute(by(t[['name', 'amount']], t.count()), bank)) == \
+            set([(d['name'], d['amount'], 1) for d in bank_raw])
+
+
+def test_datetime_handling(e, events):
+    assert set(compute(e[e.time >= datetime(2012, 1, 2, 12, 0, 0)].x,
+                        events)) == set([2, 3])
+    assert set(compute(e[e.time >= "2012-01-02"].x,
+                        events)) == set([2, 3])
+
+
+def test_summary_kwargs(t, bank):
+    expr = by(t.name, total=t.amount.sum(), avg=t.amount.mean())
+    result = compute(expr, bank)
+    assert result == [('Bob', 200.0, 600), ('Alice', 150.0, 300)]
+
+
+def test_summary_count(t, bank):
+    expr = by(t.name, how_many=t.amount.count())
+    result = compute(expr, bank)
+    assert result == [('Bob', 3), ('Alice', 2)]
+
+
+@xfail(raises=AttributeError,
+       reason='ColumnWise not implemented for MongoDB')
+def test_summary_arith(t, bank):
+    expr = by(t.name, add_one_and_sum=(t.amount + 1).sum())
+    result = compute(expr, bank)
+    assert result == [('Bob', 601), ('Alice', 301)]