Repository: blaze/blaze

Issue:
MongoDB tests fail on Windows
It's pretty easy to get a MongoDB server running on Windows using these instructions:

http://docs.mongodb.org/manual/tutorial/install-mongodb-on-windows/#manually-create-a-windows-service-for-mongodb

When I run tests I get a lot of failures. The backtrace is large so I won't put it up, but we should start testing mongo on Windows.



Discussion:
I should note that some of these errors will be fixed when the CSV pr is merged


Resolving patch:
diff --git a/blaze/api/into.py b/blaze/api/into.py
--- a/blaze/api/into.py
+++ b/blaze/api/into.py
@@ -1,11 +1,12 @@
 from __future__ import absolute_import, division, print_function
 
+import os
 from dynd import nd
 import datashape
 import sys
 from datashape import dshape, Record, to_numpy_dtype
 import toolz
-from toolz import concat, partition_all, valmap, compose
+from toolz import concat, partition_all, valmap
 from cytoolz import pluck
 import copy
 from datetime import datetime
@@ -562,7 +563,8 @@ def into(l, coll, columns=None, schema=None):
 
 @dispatch((tuple, list), Collection)
 def into(l, coll, columns=None, schema=None):
-    return type(l)(into(Iterator, coll, columns=columns, schema=schema))
+    r = into(Iterator, coll, columns=columns, schema=schema)
+    return type(l)(r)
 
 
 @dispatch(Collection, CSV)
@@ -587,70 +589,59 @@ def into(coll, d, if_exists="replace", **kwargs):
     csv_dd = d
     db = coll.database
 
-    try:
-        copy_info = {
-            'dbname':db.name,
-            'coll': coll.name,
-            'abspath': d._abspath
-            }
-        optional_flags = []
-
-        if if_exists=='replace':
-            optional_flags.append('--drop')
-
-        if kwargs.get('header',csv_dd.header):
-            optional_flags.append('--headerline')
-        if kwargs.get('ignore_blank', None):
-            optional_flags.append('--ignoreBlanks')
-
-        cols = kwargs.get('columns', csv_dd.columns)
-        copy_info['column_names'] = ','.join(cols)
-
-        if csv_dd.dialect['delimiter'] == ',':
-            copy_info['file_type'] = 'csv'
-        elif csv_dd.dialect['delimiter'] == '\t':
-            copy_info['file_type'] = 'tsv'
-        else:
-            raise NotImplementedError("Only CSV and TSV files are supported by mongoimport")
-
-        copy_cmd = """
-                mongoimport -d {dbname} \
-                 -c {coll} --type {file_type} \
-                 --file {abspath} --fields {column_names}\
-                 """
-
-        copy_cmd = copy_cmd.format(**copy_info)
-        copy_cmd = copy_cmd + ' '.join(optional_flags)
-        ps = subprocess.Popen(copy_cmd,shell=True, stdout=subprocess.PIPE)
-        output = ps.stdout.read()
-
-        #need to check for date columns and update
-        date_cols = []
-        dshape = csv_dd.dshape
-        for t, c in zip(dshape[1].types, dshape[1].names):
-            if hasattr(t, "ty"):
-                if isinstance(t.ty, datashape.Date) or isinstance(t.ty, datashape.DateTime):
-                    date_cols.append((c, t.ty))
-
-        for d_col, ty in date_cols:
-            mongo_data = list(coll.find({},{d_col:1}))
-            for doc in mongo_data:
-                try:
-                    t = parser.parse(doc[d_col])
-                except AttributeError:
-                    print(m_id, " is already of type datetime")
-                    t = doc[d_col]
-                m_id = doc['_id']
-                coll.update({'_id':m_id},{"$set": {d_col: t}})
-
-        return coll
-    except Exception as e:
-        # not sure what will go wrong yet
-        # should we roll back and drop collection?
-        print("Fast mongoimport operation failed.  Reason: ", e)
-        print("Trying again without mongoimport call")
+    copy_info = {
+        'dbname': db.name,
+        'coll': coll.name,
+        'abspath': d._abspath
+    }
+
+    optional_flags = []
+
+    if if_exists == 'replace':
+        optional_flags.append('--drop')
+
+    if kwargs.get('header', csv_dd.header):
+        optional_flags.append('--headerline')
+    if kwargs.get('ignore_blank', None):
+        optional_flags.append('--ignoreBlanks')
+
+    cols = kwargs.get('columns', csv_dd.columns)
+    copy_info['column_names'] = ','.join(cols)
+
+    delim = csv_dd.dialect['delimiter']
+    typ = copy_info['file_type'] = {',': 'csv', '\t': 'tsv'}.get(delim, None)
+    if typ is None:
         dd_into_coll = into.dispatch(Collection, DataDescriptor)
-        return dd_into_coll(coll,csv_dd)
+        return dd_into_coll(coll, csv_dd)
+
+    copy_cmd = ("mongoimport -d {dbname} -c {coll} --type {file_type} "
+                "--file {abspath} --fields {column_names} ")
+
+    copy_cmd = copy_cmd.format(**copy_info) + ' '.join(optional_flags)
+
+    ps = subprocess.Popen(copy_cmd, shell=os.name != 'nt',
+                          stdout=subprocess.PIPE)
+    ps.wait()
+
+    # need to check for date columns and update
+    date_cols = []
+    dshape = csv_dd.dshape
+    for t, c in zip(dshape[1].types, dshape[1].names):
+        if hasattr(t, "ty"):
+            if isinstance(t.ty, (datashape.Date, datashape.DateTime)):
+                date_cols.append((c, t.ty))
+
+    for d_col, ty in date_cols:
+        mongo_data = list(coll.find({}, {d_col: 1}))
+        for doc in mongo_data:
+            try:
+                t = parser.parse(doc[d_col])
+            except AttributeError:
+                t = doc[d_col]
+            m_id = doc['_id']
+            coll.update({'_id': m_id}, {"$set": {d_col: t}})
+
+    return coll
 
 
 @dispatch(Collection, (JSON, JSON_Streaming))
@@ -669,38 +660,29 @@ def into(coll, d, if_exists="replace", **kwargs):
     json_dd = d
     db = coll.database
 
-    try:
-        copy_info = {
-            'dbname':db.name,
-            'coll': coll.name,
-            'abspath': d._abspath
-            }
-        optional_flags = []
-
-        if if_exists=='replace':
-            optional_flags.append('--drop')
+    copy_info = {
+        'dbname': db.name,
+        'coll': coll.name,
+        'abspath': d._abspath
+    }
+    optional_flags = []
 
-        if kwargs.get('json_array', None):
-            optional_flags.append('--jsonArray')
+    if if_exists == 'replace':
+        optional_flags.append('--drop')
 
-        copy_info['file_type'] = 'json'
+    if kwargs.get('json_array', None):
+        optional_flags.append('--jsonArray')
 
-        copy_cmd = "mongoimport -d {dbname} -c {coll} --type {file_type} --file {abspath} "
+    copy_info['file_type'] = 'json'
 
-        copy_cmd = copy_cmd.format(**copy_info)
-        copy_cmd = copy_cmd + ' '.join(optional_flags)
-        ps = subprocess.Popen(copy_cmd,shell=True, stdout=subprocess.PIPE)
+    copy_cmd = ("mongoimport -d {dbname} -c {coll} --type {file_type} "
+                "--file {abspath} ")
 
-        output = ps.stdout.read()
+    copy_cmd = copy_cmd.format(**copy_info) + ' '.join(optional_flags)
 
-        return coll
-    except Exception as e:
-        # not sure what will go wrong yet
-        # should we roll back and drop collection?
-        print("Fast mongoimport operation failed.  Reason: ", e)
-        print("Trying again without mongoimport call")
-        dd_into_coll = into.dispatch(Collection, DataDescriptor)
-        return dd_into_coll(coll,csv_dd)
+    ps = subprocess.Popen(copy_cmd, shell=os.name != 'nt',
+                          stdout=subprocess.PIPE)
+    ps.wait()
 
 
 @dispatch(nd.array, DataDescriptor)


Test patch:
diff --git a/blaze/api/tests/test_into_exhaustive.py b/blaze/api/tests/test_into_exhaustive.py
--- a/blaze/api/tests/test_into_exhaustive.py
+++ b/blaze/api/tests/test_into_exhaustive.py
@@ -1,5 +1,6 @@
 from __future__ import absolute_import, division, print_function
 
+import pytest
 from dynd import nd
 import numpy as np
 from pandas import DataFrame
@@ -7,7 +8,6 @@
 from blaze.api.into import into, discover
 from blaze.api.into import degrade_numpy_dtype_to_python
 from datashape import dshape
-from blaze.bcolz import *
 import blaze
 from blaze import Table, TableExpr, TableSymbol, compute
 import bcolz
@@ -16,6 +16,7 @@
 from datetime import datetime
 from toolz import pluck
 import os
+from blaze.compatibility import xfail
 
 dirname = os.path.dirname(__file__)
 
@@ -177,10 +178,25 @@ def test_ColumnDataSource():
         assert into(ColumnDataSource, a).data == cds.data
 
 
-@skip_if_not(pymongo)
-def test_mongo_Collection():
+@pytest.fixture
+def mconn():
+    pymongo = pytest.importorskip('pymongo')
+    try:
+        c = pymongo.MongoClient()
+    except pymongo.errors.ConnectionFailure:
+        pytest.skip('unable to connect')
+    else:
+        return c
+
+
+@pytest.fixture
+def mdb(mconn):
+    return mconn.db
+
+
+def test_mongo_Collection(mdb):
     sources = [v for k, v in data.items() if k not in [list]]
     for a in sources:
         db.test_into.drop()
         into(db.test_into, a)
-        assert normalize(into([], db.test_into)) == normalize(L)
+        assert normalize(into([], db.test_into)) == normalize(L)
\ No newline at end of file
diff --git a/blaze/tests/test_mongo.py b/blaze/tests/test_mongo.py
--- a/blaze/tests/test_mongo.py
+++ b/blaze/tests/test_mongo.py
@@ -7,70 +7,93 @@
 import tempfile
 import json
 import os
-from blaze.utils import filetext
-
-pymongo = pytest.importorskip('pymongo')
-
-try:
-    pymongo.MongoClient()
-except pymongo.errors.ConnectionFailure:
-    pytest.importorskip('fhskjfdskfhsf')
+from blaze.utils import filetext, tmpfile, raises
+from blaze.compatibility import PY3
 
 from datashape import discover, dshape
 
 from blaze import drop, into, create_index
 
-conn = pymongo.MongoClient()
-db = conn.test_db
-file_name = 'test.csv'
-file_name_colon = 'colon_test.csv'
 
-from pymongo import ASCENDING, DESCENDING
+no_mongoimport = pytest.mark.skipif(raises(OSError,
+    lambda : subprocess.Popen('mongoimport',
+                              shell=os.name != 'nt',
+                              stdout=subprocess.PIPE).wait()),
+    reason='mongoimport cannot be found')
+
+
+@pytest.yield_fixture(scope='module')
+def conn():
+    pymongo = pytest.importorskip('pymongo')
+    try:
+        c = pymongo.MongoClient()
+    except pymongo.errors.ConnectionFailure:
+        pytest.skip('No mongo server running')
+    else:
+        yield c
+        c.close()
+
+
+@pytest.fixture
+def db(conn):
+    return conn.test_db
 
-def setup_function(function):
-    data = [(1, 2), (10, 20), (100, 200)]
 
-    with open(file_name, 'w') as f:
-        csv_writer = csv_module.writer(f)
-        for row in data:
-            csv_writer.writerow(row)
+@pytest.fixture
+def tuple_data():
+    return [(1, 2), (10, 20), (100, 200)]
 
-    with open(file_name_colon, 'w') as f:
-        csv_writer = csv_module.writer(f, delimiter=':')
-        for row in data:
-            csv_writer.writerow(row)
 
+@pytest.fixture
+def newline():
+    return {'newline': '' if os.name == 'nt' else None} if PY3 else {}
 
-def teardown_function(function):
-    os.remove(file_name)
-    os.remove(file_name_colon)
+
+@pytest.yield_fixture
+def file_name_colon(tuple_data, newline):
+    with tmpfile('.csv') as filename:
+        with open(filename, 'w', **newline) as f:
+            csv_writer = csv_module.writer(f, delimiter=':')
+            csv_writer.writerows(tuple_data)
+        yield filename
 
 
 @pytest.yield_fixture
-def empty_collec():
+def file_name(tuple_data, newline):
+    with tmpfile('.csv') as filename:
+        with open(filename, 'w', **newline) as f:
+            csv_writer = csv_module.writer(f)
+            csv_writer.writerows(tuple_data)
+        yield filename
+
+
+@pytest.yield_fixture
+def empty_collec(db):
     yield db.tmp_collection
     db.tmp_collection.drop()
 
 
+@pytest.fixture
+def bank():
+    return [{'name': 'Alice', 'amount': 100},
+            {'name': 'Alice', 'amount': 200},
+            {'name': 'Bob', 'amount': 100},
+            {'name': 'Bob', 'amount': 200},
+            {'name': 'Bob', 'amount': 300}]
+
+
 @pytest.yield_fixture
-def bank_collec():
+def bank_collec(db, bank):
     coll = into(db.tmp_collection, bank)
     yield coll
     coll.drop()
 
 
-bank = [{'name': 'Alice', 'amount': 100},
-        {'name': 'Alice', 'amount': 200},
-        {'name': 'Bob', 'amount': 100},
-        {'name': 'Bob', 'amount': 200},
-        {'name': 'Bob', 'amount': 300}]
-
-
 def test_discover(bank_collec):
     assert discover(bank_collec) == dshape('5 * {amount: int64, name: string}')
 
 
-def test_into(empty_collec):
+def test_into(empty_collec, bank):
     lhs = set(into([], into(empty_collec, bank), columns=['name', 'amount']))
     rhs = set([('Alice', 100), ('Alice', 200), ('Bob', 100), ('Bob', 200),
                ('Bob', 300)])
@@ -78,13 +101,10 @@ def test_into(empty_collec):
 
 
 @pytest.yield_fixture
-def mongo():
-    conn = pymongo.MongoClient()
-    db = conn.db
+def mongo(db, bank):
     db.tmp_collection.insert(bank)
     yield db
     db.tmp_collection.drop()
-    conn.close()
 
 
 def test_drop(mongo):
@@ -92,7 +112,9 @@ def test_drop(mongo):
     assert mongo.tmp_collection.count() == 0
 
 
-bank_idx = [{'name': 'Alice', 'amount': 100, 'id': 1},
+@pytest.fixture
+def bank_idx():
+    return [{'name': 'Alice', 'amount': 100, 'id': 1},
             {'name': 'Alice', 'amount': 200, 'id': 2},
             {'name': 'Bob', 'amount': 100, 'id': 3},
             {'name': 'Bob', 'amount': 200, 'id': 4},
@@ -100,14 +122,10 @@ def test_drop(mongo):
 
 
 @pytest.yield_fixture
-def mongo_idx():
-    pymongo = pytest.importorskip('pymongo')
-    conn = pymongo.MongoClient()
-    db = conn.db
+def mongo_idx(db, bank_idx):
     db.tmp_collection.insert(bank_idx)
     yield db
     db.tmp_collection.drop()
-    conn.close()
 
 
 class TestCreateIndex(object):
@@ -124,11 +142,13 @@ def test_create_composite_index(self, mongo_idx):
         assert 'id_1_amount_1' in mongo_idx.tmp_collection.index_information()
 
     def test_create_composite_index_params(self, mongo_idx):
+        from pymongo import ASCENDING, DESCENDING
         create_index(mongo_idx.tmp_collection,
                      [('id', ASCENDING), ('amount', DESCENDING)])
         assert 'id_1_amount_-1' in mongo_idx.tmp_collection.index_information()
 
     def test_fails_when_using_not_list_of_tuples_or_strings(self, mongo_idx):
+        from pymongo import DESCENDING
         with pytest.raises(TypeError):
             create_index(mongo_idx.tmp_collection, [['id', DESCENDING]])
 
@@ -152,12 +172,14 @@ def test_create_composite_index(self, mongo_idx):
         assert 'c_idx' in mongo_idx.tmp_collection.index_information()
 
     def test_create_composite_index_params(self, mongo_idx):
+        from pymongo import ASCENDING, DESCENDING
         create_index(mongo_idx.tmp_collection,
                      [('id', ASCENDING), ('amount', DESCENDING)],
                      name='c_idx')
         assert 'c_idx' in mongo_idx.tmp_collection.index_information()
 
     def test_fails_when_using_not_list_of_tuples_or_strings(self, mongo_idx):
+        from pymongo import DESCENDING
         with pytest.raises(TypeError):
             create_index(mongo_idx.tmp_collection, [['id', DESCENDING]])
 
@@ -167,7 +189,8 @@ def test_create_index_with_unique(self, mongo_idx):
         assert coll.index_information()['c_idx']['unique']
 
 
-def test_csv_mongodb_load(empty_collec):
+@no_mongoimport
+def test_csv_mongodb_load(db, file_name, empty_collec):
 
     csv = CSV(file_name)
 
@@ -185,53 +208,53 @@ def test_csv_mongodb_load(empty_collec):
         'column_names': ','.join(csv.columns)
     }
 
-    copy_cmd = """
-                mongoimport -d {dbname} -c {coll} --type csv --file {abspath} --fields {column_names}
-               """
+    copy_cmd = """mongoimport -d {dbname} -c {coll} --type csv --file {abspath} --fields {column_names}"""
     copy_cmd = copy_cmd.format(**copy_info)
 
-    ps = subprocess.Popen(copy_cmd,shell=True, stdout=subprocess.PIPE)
+    ps = subprocess.Popen(copy_cmd, shell=os.name != 'nt',
+                          stdout=subprocess.PIPE)
     output = ps.stdout.read()
-    mongo_data = list(coll.find({},{'_0': 1, '_id': 0}))
+    mongo_data = list(coll.find({}, {'_0': 1, '_id': 0}))
 
-    assert list(csv[:,'_0']) == [i['_0'] for i in mongo_data]
+    assert list(csv[:, '_0']) == [i['_0'] for i in mongo_data]
 
 
-def test_csv_into_mongodb(empty_collec):
-    csv = CSV(file_name)
-
+def test_csv_into_mongodb_colon_del(empty_collec, file_name_colon):
+    csv = CSV(file_name_colon)
     coll = empty_collec
-    into(coll,csv)
-    mongo_data = list(coll.find({},{'_0': 1, '_id': 0}))
-
-    assert list(csv[:,'_0']) == [i['_0'] for i in mongo_data]
-
+    lhs = into(list, csv)
+    newcoll = into(coll, csv)
+    rhs = into(list, newcoll)
+    assert lhs == rhs
 
-def test_csv_into_mongodb_colon_del(empty_collec):
-    csv = CSV(file_name_colon)
 
+def test_csv_into_mongodb(empty_collec, file_name):
+    csv = CSV(file_name)
     coll = empty_collec
+    res = into(coll, csv)
+    mongo_data = list(res.find({}, {'_0': 1, '_id': 0}))
 
-    assert into(list, csv) == into(list, into(coll, csv))
+    assert list(csv[:, '_0']) == [i['_0'] for i in mongo_data]
 
 
-def test_csv_into_mongodb_columns(empty_collec):
+def test_csv_into_mongodb_columns(empty_collec, file_name):
     csv = CSV(file_name, schema='{x: int, y: int}')
 
     coll = empty_collec
 
     assert into(list, csv) == into(list, into(coll, csv))
 
+
 def test_csv_into_mongodb_complex(empty_collec):
 
     this_dir = os.path.dirname(__file__)
     file_name = os.path.join(this_dir, 'dummydata.csv')
 
-    csv = CSV(file_name, schema = "{ Name : string, RegistrationDate : ?datetime, ZipCode : ?int64, Consts : ?float64 }")
+    csv = CSV(file_name, schema="{ Name : string, RegistrationDate : ?datetime, ZipCode : ?int64, Consts : ?float64 }")
     coll = empty_collec
-    into(coll,csv)
+    into(coll, csv)
 
-    mongo_data = list(coll.find({},{'_id': 0}))
+    mongo_data = list(coll.find({}, {'_id': 0}))
 
     # This assertion doesn't work due to python floating errors
     # into(list, csv) == into(list, into(coll, csv))
@@ -251,6 +274,7 @@ def test_csv_into_mongodb_complex(empty_collec):
                 }
 
 
+@no_mongoimport
 def test_json_into_mongodb(empty_collec):
 
     with filetext(json.dumps(les_mis_data)) as filename:
@@ -295,18 +319,19 @@ def test_json_into_mongodb(empty_collec):
          u'tv_show': u'Battlestar Galactica'}]
 
 
+@no_mongoimport
 def test_jsonarray_into_mongodb(empty_collec):
 
     filename = tempfile.mktemp(".json")
     with open(filename, "w") as f:
         json.dump(data, f)
 
-    dd = JSON(filename, schema = "3 * { id : string, name : string, posts : var * { content : string, title : string },\
-                                 tv_show : string }")
+    dd = JSON(filename, schema="3 * { id : string, name : string, "
+                                "posts : var * { content : string, title : string },"
+                                " tv_show : string }")
     coll = empty_collec
     into(coll,dd, json_array=True)
 
-
-    mongo_data = list(coll.find({},{'_id': 0,}))
+    mongo_data = list(coll.find({}, {'_id': 0}))
 
     assert mongo_data[0] == data[0]