Repository: blaze/blaze

Issue:
Support Mongo Databases as first class citizens
A while ago we started supporting database like objects as data.  For example with SQLAlchemy we have implementations like 

```
@dispatch(Field, Engine)
def compute_up(expr, data, **kwargs):
    ...
```

And it's now common to give an engine to `Data` and use field access to navigate various tables.

We left behind Mongo when we did this and it has now caught up to us in #941 .  In particular `Data('...::mytable')` now _assumes_ that it can turn this in to `Data('...').mytable`.

So, it would be nice if we knew how to operate on Mongo Databases themselves, not just collections.  This generally requires making a `compute_up` function that selects a collection given a database (easy) and a `discover` function which finds all of the collections of a database (also usually easy).  When looking a little while ago at the pymongo api I didn't find a "list all of the valid collections" method, but presumably there is some way to do this.



Discussion:

Resolving patch:
diff --git a/blaze/compute/mongo.py b/blaze/compute/mongo.py
--- a/blaze/compute/mongo.py
+++ b/blaze/compute/mongo.py
@@ -48,6 +48,7 @@
 
 try:
     from pymongo.collection import Collection
+    from pymongo.database import Database
 except ImportError:
     Collection = type(None)
 
@@ -57,27 +58,33 @@
 import toolz
 import datetime
 
-from ..expr import (var, Label, std, Sort, count, nunique, nelements, Selection,
+from ..expr import (Sort, count, nunique, nelements, Selection,
                     mean, Reduction, Head, ReLabel, Distinct, ElemWise, By,
                     Symbol, Projection, Field, sum, min, max, Gt, Lt, Ge, Le,
                     Eq, Ne, And, Or, Summary, Like, Broadcast, DateTime,
                     Microsecond, Date, Time, Expr, symbol, Arithmetic, floor,
                     ceil, FloorDiv)
+from ..expr.broadcast import broadcast_collect
 from ..expr import math
 from ..expr.datetime import Day, Month, Year, Minute, Second, UTCFromTimestamp
 from ..compatibility import _strtypes
-from .core import compute
 
 from ..dispatch import dispatch
 
 
 __all__ = ['MongoQuery']
 
+
 @dispatch(Expr, Collection)
 def pre_compute(expr, data, scope=None, **kwargs):
     return MongoQuery(data, [])
 
 
+@dispatch(Expr, Database)
+def pre_compute(expr, data, **kwargs):
+    return data
+
+
 class MongoQuery(object):
     """
     A Pair of a pymongo collection and a aggregate query
@@ -114,11 +121,9 @@ def __hash__(self):
         return hash((type(self), self.info()))
 
 
-from ..expr.broadcast import broadcast_collect
-
 @dispatch(Expr, (MongoQuery, Collection))
 def optimize(expr, seq):
-    return broadcast_collect( expr)
+    return broadcast_collect(expr)
 
 
 @dispatch(Head, MongoQuery)
@@ -347,6 +352,11 @@ def compute_up(t, q, **kwargs):
                   Second: 'second'}
 
 
+@dispatch(Field, Database)
+def compute_up(expr, data, **kwargs):
+    return getattr(data, expr._name)
+
+
 @dispatch(Expr, Collection)
 def post_compute(e, c, scope=None):
     """
diff --git a/blaze/interactive.py b/blaze/interactive.py
--- a/blaze/interactive.py
+++ b/blaze/interactive.py
@@ -31,6 +31,7 @@
 try:
     import pymongo
     not_an_iterator.append(pymongo.collection.Collection)
+    not_an_iterator.append(pymongo.database.Database)
 except ImportError:
     pass
 
diff --git a/blaze/mongo.py b/blaze/mongo.py
--- a/blaze/mongo.py
+++ b/blaze/mongo.py
@@ -1,14 +1,9 @@
 from __future__ import absolute_import, division, print_function
 
-import re
-from toolz import take
-from datashape import discover, isdimension
-
-from .compatibility import basestring, map
+from .compatibility import basestring
 from .compute.mongo import dispatch
 
 try:
-    import pymongo
     from pymongo.collection import Collection
     from pymongo import ASCENDING
 except ImportError:


Test patch:
diff --git a/blaze/compute/tests/test_mongo_compute.py b/blaze/compute/tests/test_mongo_compute.py
--- a/blaze/compute/tests/test_mongo_compute.py
+++ b/blaze/compute/tests/test_mongo_compute.py
@@ -1,13 +1,13 @@
 from __future__ import absolute_import, division, print_function
 
-from operator import itemgetter
 import pytest
 pymongo = pytest.importorskip('pymongo')
 
 from datetime import datetime
 from toolz import pluck, reduceby, groupby
 
-from blaze import into, compute, compute_up, discover, dshape
+from datashape import Record
+from blaze import into, compute, compute_up, discover, dshape, Data
 
 from blaze.compute.mongo import MongoQuery
 from blaze.expr import symbol, by, floor, ceil
@@ -49,8 +49,6 @@ def big_bank(db):
 
 @pytest.yield_fixture
 def date_data(db):
-    import numpy as np
-    import pandas as pd
     n = 3
     d = {'name': ['Alice', 'Bob', 'Joe'],
          'when': [datetime(2010, 1, 1, i) for i in [1, 2, 3]],
@@ -58,14 +56,14 @@ def date_data(db):
          'id': [1, 2, 3]}
     data = [dict(zip(d.keys(), [d[k][i] for k in d.keys()]))
             for i in range(n)]
-    coll = into(db.date_coll, data)
+    coll = into(db.date_data, data)
     yield coll
     coll.drop()
 
 
 @pytest.yield_fixture
 def bank(db):
-    coll = db.tmp_collection
+    coll = db.bank
     coll = into(coll, bank_raw)
     yield coll
     coll.drop()
@@ -77,7 +75,7 @@ def missing_vals(db):
             {'x': 2, 'y': 20, 'z': 200},
             {'x': 3, 'z': 300},
             {'x': 4, 'y': 40}]
-    coll = db.tmp_collection
+    coll = db.missing_vals
     coll = into(coll, data)
     yield coll
     coll.drop()
@@ -89,7 +87,7 @@ def points(db):
             {'x': 2, 'y': 20, 'z': 200},
             {'x': 3, 'y': 30, 'z': 300},
             {'x': 4, 'y': 40, 'z': 400}]
-    coll = db.tmp_collection
+    coll = db.points
     coll = into(coll, data)
     yield coll
     coll.drop()
@@ -100,7 +98,7 @@ def events(db):
     data = [{'time': datetime(2012, 1, 1, 12, 00, 00), 'x': 1},
             {'time': datetime(2012, 1, 2, 12, 00, 00), 'x': 2},
             {'time': datetime(2012, 1, 3, 12, 00, 00), 'x': 3}]
-    coll = db.tmp_collection
+    coll = db.events
     coll = into(coll, data)
     yield coll
     coll.drop()
@@ -115,6 +113,15 @@ def events(db):
 q = MongoQuery('fake', [])
 
 
+def test_compute_on_db(bank, points):
+    assert bank.database == points.database
+    db = bank.database
+
+    d = symbol(db.name, discover(db))
+    assert (compute(d.points.x.sum(), db) ==
+            sum(x['x'] for x in db.points.find()))
+
+
 def test_symbol(bank):
     assert compute(t, bank) == list(pluck(['name', 'amount'], bank_raw))
 
@@ -137,21 +144,21 @@ def test_projection(bank):
 
 
 def test_selection(bank):
-    assert set(compute(t[t.name=='Alice'], bank)) == set([('Alice', 100),
+    assert set(compute(t[t.name == 'Alice'], bank)) == set([('Alice', 100),
                                                             ('Alice', 200)])
-    assert set(compute(t['Alice'==t.name], bank)) == set([('Alice', 100),
+    assert set(compute(t['Alice' == t.name], bank)) == set([('Alice', 100),
                                                             ('Alice', 200)])
     assert set(compute(t[t.amount > 200], bank)) == set([('Bob', 300)])
     assert set(compute(t[t.amount >= 200], bank)) == set([('Bob', 300),
                                                           ('Bob', 200),
                                                           ('Alice', 200)])
-    assert set(compute(t[t.name!='Alice'].name, bank)) == set(['Bob'])
-    assert set(compute(t[(t.name=='Alice') & (t.amount > 150)], bank)) == \
-            set([('Alice', 200)])
-    assert set(compute(t[(t.name=='Alice') | (t.amount > 250)], bank)) == \
-            set([('Alice', 200),
-                    ('Alice', 100),
-                    ('Bob', 300)])
+    assert set(compute(t[t.name != 'Alice'].name, bank)) == set(['Bob'])
+    assert set(compute(t[(t.name == 'Alice') & (t.amount > 150)], bank)) == \
+        set([('Alice', 200)])
+    assert set(compute(t[(t.name == 'Alice') | (t.amount > 250)], bank)) == \
+        set([('Alice', 200),
+             ('Alice', 100),
+             ('Bob', 300)])
 
 
 def test_columnwise(points):
@@ -184,20 +191,20 @@ def test_columnwise_pow(points):
 
 def test_by_one():
     assert compute_up(by(t.name, total=t.amount.sum()), q).query == \
-            ({'$group': {'_id': {'name': '$name'},
-                         'total': {'$sum': '$amount'}}},
-             {'$project': {'total': '$total', 'name': '$_id.name'}})
+        ({'$group': {'_id': {'name': '$name'},
+                     'total': {'$sum': '$amount'}}},
+         {'$project': {'total': '$total', 'name': '$_id.name'}})
 
 
 def test_by(bank):
     assert set(compute(by(t.name, total=t.amount.sum()), bank)) == \
-            set([('Alice', 300), ('Bob', 600)])
+        set([('Alice', 300), ('Bob', 600)])
     assert set(compute(by(t.name, min=t.amount.min()), bank)) == \
-            set([('Alice', 100), ('Bob', 100)])
+        set([('Alice', 100), ('Bob', 100)])
     assert set(compute(by(t.name, max=t.amount.max()), bank)) == \
-            set([('Alice', 200), ('Bob', 300)])
+        set([('Alice', 200), ('Bob', 300)])
     assert set(compute(by(t.name, count=t.name.count()), bank)) == \
-            set([('Alice', 2), ('Bob', 3)])
+        set([('Alice', 2), ('Bob', 3)])
 
 
 def test_reductions(bank):
@@ -212,21 +219,21 @@ def test_distinct(bank):
 
 def test_sort(bank):
     assert compute(t.amount.sort('amount'), bank) == \
-            [100, 100, 200, 200, 300]
+        [100, 100, 200, 200, 300]
     assert compute(t.amount.sort('amount', ascending=False), bank) == \
-            [300, 200, 200, 100, 100]
+        [300, 200, 200, 100, 100]
 
 
 def test_by_multi_column(bank):
     assert set(compute(by(t[['name', 'amount']], count=t.count()), bank)) == \
-            set([(d['name'], d['amount'], 1) for d in bank_raw])
+        set([(d['name'], d['amount'], 1) for d in bank_raw])
 
 
 def test_datetime_handling(events):
     assert set(compute(e[e.time >= datetime(2012, 1, 2, 12, 0, 0)].x,
-                        events)) == set([2, 3])
+                       events)) == set([2, 3])
     assert set(compute(e[e.time >= "2012-01-02"].x,
-                        events)) == set([2, 3])
+                       events)) == set([2, 3])
 
 
 def test_summary_kwargs(bank):
@@ -278,7 +285,8 @@ def test_summary_complex_arith_multiple(bank):
     values = list(mu.values())
     items = expected.items()
     counts = groupby('name', bank.find())
-    items = [x + (float(v) / len(counts[x[0]]),) for x, v in zip(items, values)]
+    items = [x + (float(v) / len(counts[x[0]]),)
+             for x, v in zip(items, values)]
     assert set(result) == set(items)
 
 
@@ -305,27 +313,27 @@ def test_like_mulitple_no_match(big_bank):
 
 def test_missing_values(missing_vals):
     assert discover(missing_vals).subshape[0] == \
-            dshape('{x: int64, y: ?int64, z: ?int64}')
+        dshape('{x: int64, y: ?int64, z: ?int64}')
 
     assert set(compute(p.y, missing_vals)) == set([None, 20, None, 40])
 
 
 def test_datetime_access(date_data):
     t = symbol('t',
-            'var * {amount: float64, id: int64, name: string, when: datetime}')
+               'var * {amount: float64, id: int64, name: string, when: datetime}')
 
-    py_data = into(list, date_data) # a python version of the collection
+    py_data = into(list, date_data)  # a python version of the collection
 
     for attr in ['day', 'minute', 'second', 'year', 'month']:
         assert list(compute(getattr(t.when, attr), date_data)) == \
-                list(compute(getattr(t.when, attr), py_data))
+            list(compute(getattr(t.when, attr), py_data))
 
 
 def test_datetime_access_and_arithmetic(date_data):
     t = symbol('t',
-            'var * {amount: float64, id: int64, name: string, when: datetime}')
+               'var * {amount: float64, id: int64, name: string, when: datetime}')
 
-    py_data = into(list, date_data) # a python version of the collection
+    py_data = into(list, date_data)  # a python version of the collection
 
     expr = t.when.day + t.id
 
@@ -336,3 +344,8 @@ def test_floor_ceil(bank):
     t = symbol('t', discover(bank))
     assert set(compute(200 * floor(t.amount / 200), bank)) == set([0, 200])
     assert set(compute(200 * ceil(t.amount / 200), bank)) == set([200, 400])
+
+
+def test_Data_construct():
+    d = Data('mongodb://localhost/test_db')
+    assert isinstance(d.dshape.measure, Record)