Repository: blaze/blaze

Issue:
Error when loading MongoDB collection
Hi there,

I'm getting the error shown below when trying load a MongoDB collection.

`data = blaze.Data('mongodb://localhost/mydb::mycollection')`

I'm using the latest github version. The same command works with 0.7.0. Any ideas?

Thanks,
Sigurd

```
Traceback (most recent call last):
  File "<stdin>", line 1, in <module>
  File "/home/sigurd/.local/lib/python2.7/site-packages/blaze-0.7.1-py2.7.egg/blaze/interactive.py", line 45, in Data
    columns=columns, **kwargs)
  File "/home/sigurd/.local/lib/python2.7/site-packages/into-0.1.3-py2.7.egg/into/regex.py", line 64, in __call__
    return self.dispatch(s)(s, *args, **kwargs)
TypeError: resource_mongo() takes exactly 2 arguments (1 given)
```



Discussion:
Yeah, I ran into this myself recently.  Hope to fix it in the next week or so.  Until then here is a work-around

```
data = blaze.Data(blaze.resource('mongodb://localhost/mydb::mycollection'))
```

Is it an easy fix? I'm new to Blaze's internals, so if you could guide me through the process, I'd be happy to provide a fix.

Ah, sorry for letting this hang.  In short the solution is that we need to teach blaze+into how to interact with Mongo Databases themselves rather than just individual collections.  Generally Blaze considers a database of tables to be something like a dictionary of tables.  This lets us use field/column access to get at particular tables e.g. `db.mytable`.  I think that the solution should be the following:
1.  Add a resource function for MongoDB databases (e.g. `mongodb://localhost/mydb`) to complement the existing resource function for collections (e.g. `mongodb://localhost/mydb::mycollection`).  This should happen in [into](https://github.com/ContinuumIO/into/blob/master/into/backends/mongo.py#L81-L101).  
2.  Add a similar function for discover.  This would look something like [this equivalent function for SQL metadata objects](https://github.com/ContinuumIO/into/blob/master/into/backends/sql.py#L127-L145)
3.  Teach Blaze to translate `Field` access on Mongo DataBases into getting out the right collection.  This should look similar to [the equivalent function for SQL metadata objects](https://github.com/ContinuumIO/blaze/blob/master/blaze/compute/sql.py#L568-L570)

At the end of these three steps we should have something like the following:

``` python
db = Data('mongodb://localhost/db')
db.  <tab>  
see list of collections

coll = db.mycollection  # a particular collection
```

We started doing this for SQL, HDF5 and other similar systems that had subcollections like tables and it made a lot of things much simpler.  Sadly we forgot about poor mongo in this process.

Is this the same thing? The error messages looks different (using blaze 0.7.3)

``` python
bz.Data('mongodb://localhost/planet_four::planet_four_subjects')

---------------------------------------------------------------------------
TypeError                                 Traceback (most recent call last)
<ipython-input-9-31e65e21a64a> in <module>()
----> 1 bz.Data('mongodb://localhost/planet_four::planet_four_subjects')

/Users/klay6683/miniconda3/lib/python3.4/site-packages/blaze/interactive.py in Data(data, dshape, name, fields, columns, schema, **kwargs)
     44             data, sub_uri = data.split('::')
     45         data = resource(data, schema=schema, dshape=dshape, columns=columns,
---> 46                         **kwargs)
     47     if (isinstance(data, Iterator) and
     48             not isinstance(data, tuple(not_an_iterator))):

/Users/klay6683/miniconda3/lib/python3.4/site-packages/odo/regex.py in __call__(self, s, *args, **kwargs)
     62 
     63     def __call__(self, s, *args, **kwargs):
---> 64         return self.dispatch(s)(s, *args, **kwargs)
     65 
     66     @property

TypeError: resource_mongo() missing 1 required positional argument: 'collection_name'
```

Yes, this is the same problem.

Is there no current workaround? I would like to convert a Mongo database into a pandas HDF but don't see a path using `odo` without it accepting any Mongo database URL? Or could I work around somehow using the .bson/.json files I have from the Mongo database dumps?

I don't think odo has this problem.  This is only a problem when using `Data`.  In that the case the workaround is to use Data(resource('...'))

@cpcloud is this fixed by #1013 ?

It should be. I'm afk for the rest of the night. I can check tomorrow for sure and close or fix accordingly. 


Resolving patch:
diff --git a/blaze/interactive.py b/blaze/interactive.py
--- a/blaze/interactive.py
+++ b/blaze/interactive.py
@@ -2,38 +2,44 @@
 
 import datashape
 import datetime
+import operator
+import itertools
+import warnings
+
+from collections import Iterator
+from functools import reduce
+
 from datashape import discover, Tuple, Record, DataShape, var
 from datashape.predicates import iscollection, isscalar, isrecord, istabular
+
 from pandas import DataFrame, Series
-import itertools
-from functools import reduce
+
 import numpy as np
-import warnings
-from collections import Iterator
+
+from odo import resource, odo
+from odo.utils import ignoring
 
 from .expr import Expr, Symbol, ndim
 from .dispatch import dispatch
-from odo import into, resource
 from .compatibility import _strtypes
 
-__all__ = ['Data', 'Table', 'into', 'to_html']
 
-names = ('_%d' % i for i in itertools.count(1))
+__all__ = ['Data', 'Table', 'into', 'to_html']
 
 
+names = ('_%d' % i for i in itertools.count(1))
 not_an_iterator = []
 
-try:
+
+with ignoring(ImportError):
     import bcolz
     not_an_iterator.append(bcolz.carray)
-except ImportError:
-    pass
-try:
+
+
+with ignoring(ImportError):
     import pymongo
     not_an_iterator.append(pymongo.collection.Collection)
     not_an_iterator.append(pymongo.database.Database)
-except ImportError:
-    pass
 
 
 def Data(data, dshape=None, name=None, fields=None, columns=None, schema=None,
@@ -87,7 +93,7 @@ def Data(data, dshape=None, name=None, fields=None, columns=None, schema=None,
 
 
 class InteractiveSymbol(Symbol):
-    """ Interactive data
+    """Interactive data.
 
     The ``Data`` object presents a familiar view onto a variety of forms of
     data.  This user-level object provides an interactive experience to using
@@ -95,26 +101,23 @@ class InteractiveSymbol(Symbol):
 
     Parameters
     ----------
-
-    data: anything
+    data : object
         Any type with ``discover`` and ``compute`` implementations
-    fields: list of strings - optional
+    fields : list, optional
         Field or column names, will be inferred from datasource if possible
-    dshape: string or DataShape - optional
-        Datashape describing input data
-    name: string - optional
-        A name for the table
+    dshape : str or DataShape, optional
+        DataShape describing input data
+    name : str, optional
+        A name for the data.
 
     Examples
     --------
-
     >>> t = Data([(1, 'Alice', 100),
     ...           (2, 'Bob', -200),
     ...           (3, 'Charlie', 300),
     ...           (4, 'Denis', 400),
     ...           (5, 'Edith', -500)],
     ...          fields=['id', 'name', 'balance'])
-
     >>> t[t.balance < 0].name
         name
     0    Bob
@@ -134,13 +137,16 @@ def _resources(self):
 
     @property
     def _args(self):
-        return (id(self.data), self.dshape, self._name)
+        return id(self.data), self.dshape, self._name
 
     def __setstate__(self, state):
         for slot, arg in zip(self.__slots__, state):
             setattr(self, slot, arg)
 
 
+Data.__doc__ = InteractiveSymbol.__doc__
+
+
 def Table(*args, **kwargs):
     """ Deprecated, see Data instead """
     warnings.warn("Table is deprecated, use Data instead",
@@ -172,11 +178,11 @@ def concrete_head(expr, n=10):
     head = expr.head(n + 1)
 
     if not iscollection(expr.dshape):
-        return into(object, head)
+        return odo(head, object)
     elif isrecord(expr.dshape.measure):
-        return into(DataFrame, head)
+        return odo(head, DataFrame)
     else:
-        df = into(DataFrame, head)
+        df = odo(head, DataFrame)
         df.columns = [expr._name]
         return df
     result = compute(head)
@@ -184,9 +190,9 @@ def concrete_head(expr, n=10):
     if len(result) == 0:
         return DataFrame(columns=expr.fields)
     if isrecord(expr.dshape.measure):
-        return into(DataFrame, result, dshape=expr.dshape)
+        return odo(result, DataFrame, dshape=expr.dshape)
     else:
-        df = into(DataFrame, result, dshape=expr.dshape)
+        df = odo(result, DataFrame, dshape=expr.dshape)
         df.columns = [expr._name]
         return df
 
@@ -197,7 +203,6 @@ def repr_tables(expr, n=10):
     if isinstance(result, (DataFrame, Series)):
         s = repr(result)
         if len(result) > 10:
-            result = result[:10]
             s = '\n'.join(s.split('\n')[:-1]) + '\n...'
         return s
     else:
@@ -209,7 +214,7 @@ def numel(shape):
         return None
     if not shape:
         return 1
-    return reduce(lambda x, y: x * y, shape, 1)
+    return reduce(operator.mul, shape, 1)
 
 
 def short_dshape(ds, nlines=5):
@@ -224,7 +229,7 @@ def coerce_to(typ, x):
     try:
         return typ(x)
     except TypeError:
-        return into(typ, x)
+        return odo(x, typ)
 
 
 def coerce_scalar(result, dshape):
@@ -317,7 +322,7 @@ def table_length(expr):
 
 def intonumpy(data, dtype=None, **kwargs):
     # TODO: Don't ignore other kwargs like copy
-    result = into(np.ndarray, data)
+    result = odo(data, np.ndarray)
     if dtype and result.dtype != dtype:
         result = result.astype(dtype)
     return result
@@ -328,7 +333,7 @@ def convert_base(typ, x):
     try:
         return typ(x)
     except:
-        return typ(into(typ, x))
+        return typ(odo(x, typ))
 
 Expr.__array__ = intonumpy
 Expr.__int__ = lambda x: convert_base(int, x)


Test patch:
diff --git a/blaze/compute/tests/test_mongo_compute.py b/blaze/compute/tests/test_mongo_compute.py
--- a/blaze/compute/tests/test_mongo_compute.py
+++ b/blaze/compute/tests/test_mongo_compute.py
@@ -346,6 +346,14 @@ def test_floor_ceil(bank):
     assert set(compute(200 * ceil(t.amount / 200), bank)) == set([200, 400])
 
 
-def test_Data_construct():
+def test_Data_construct(bank, points):
     d = Data('mongodb://localhost/test_db')
+    assert 'bank' in d.fields
+    assert 'points' in d.fields
     assert isinstance(d.dshape.measure, Record)
+
+
+def test_Data_construct_with_table(bank):
+    d = Data('mongodb://localhost/test_db::bank')
+    assert set(d.fields) == set(('name', 'amount'))
+    assert int(d.count()) == 5