Repository: blaze/blaze

Issue:
CSV_DDesc does not respect its own dialect
We go through a fair amount of trouble to obtain a dialect and whether or not there is a header in the csv file, we then fail to use any of this information in our various access methods.



Discussion:

Resolving patch:
diff --git a/blaze/datadescriptor/csv_data_descriptor.py b/blaze/datadescriptor/csv_data_descriptor.py
--- a/blaze/datadescriptor/csv_data_descriptor.py
+++ b/blaze/datadescriptor/csv_data_descriptor.py
@@ -20,17 +20,17 @@ def open_file(path, mode, has_header):
     return csvfile
 
 
-def csv_descriptor_iter(filename, mode, has_header, schema):
+def csv_descriptor_iter(filename, mode, has_header, schema, dialect={}):
     with open_file(filename, mode, has_header) as csvfile:
-        for row in csv.reader(csvfile):
+        for row in csv.reader(csvfile, **dialect):
             yield DyND_DDesc(nd.array(row, dtype=schema))
 
 
 def csv_descriptor_iterchunks(filename, mode, has_header, schema,
-                              blen, start=None, stop=None):
+                              blen, dialect={}, start=None, stop=None):
     rows = []
     with open_file(filename, mode, has_header) as csvfile:
-        for nrow, row in enumerate(csv.reader(csvfile)):
+        for nrow, row in enumerate(csv.reader(csvfile, **dialect)):
             if start is not None and nrow < start:
                 continue
             if stop is not None and nrow >= stop:
@@ -64,16 +64,16 @@ class CSV_DDesc(DDesc):
         is guessed.
     """
 
-    def __init__(self, path, mode='r', **kwargs):
+    def __init__(self, path, mode='r', schema=None, dialect=None,
+            has_header=None, **kwargs):
         if os.path.isfile(path) is not True:
             raise ValueError('CSV file "%s" does not exist' % path)
         self.path = path
         self.mode = mode
         csvfile = open(path, mode=self.mode)
-        schema = kwargs.get("schema", None)
-        dialect = kwargs.get("dialect", None)
-        has_header = kwargs.get("has_header", None)
-        if type(schema) in py2help._strtypes:
+
+        # Handle Schema
+        if isinstance(schema, py2help._strtypes):
             schema = datashape.dshape(schema)
         if isinstance(schema, datashape.DataShape) and len(schema) == 1:
             schema = schema[0]
@@ -82,28 +82,39 @@ def __init__(self, path, mode='r', **kwargs):
                 'schema cannot be converted into a blaze record dshape')
         self.schema = str(schema)
 
+        # Handle Dialect
         if dialect is None:
             # Guess the dialect
             sniffer = csv.Sniffer()
             try:
-                self.dialect = sniffer.sniff(csvfile.read(1024))
+                dialect = sniffer.sniff(csvfile.read(1024))
             except:
                 # Cannot guess dialect.  Assume Excel.
-                self.dialect = csv.get_dialect('excel')
+                dialect = csv.get_dialect('excel')
             csvfile.seek(0)
         else:
-            if isinstance(dialect, csv.Dialect):
-                self.dialect = dialect
-            else:
-                self.dialect = csv.get_dialect(dialect)
+            dialect = csv.get_dialect(dialect)
+        self.dialect = dict((key, getattr(dialect, key))
+                            for key in dir(dialect) if not key.startswith('_'))
+
+        # Update dialect with any keyword arguments passed in
+        # E.g. allow user to override with delimiter=','
+        for k, v in kwargs.items():
+            if k in self.dialect:
+                self.dialect[k] = v
+
+        # Handle Header
         if has_header is None:
             # Guess whether the file has a header or not
             sniffer = csv.Sniffer()
-            self.has_header = sniffer.has_header(csvfile.read(1024))
             csvfile.seek(0)
+            sample = csvfile.read(1024)
+            self.has_header = sniffer.has_header(sample)
         else:
             self.has_header = has_header
 
+        csvfile.close()
+
     @property
     def dshape(self):
         return datashape.DataShape(datashape.Var(), self.schema)
@@ -126,7 +137,8 @@ def capabilities(self):
     def dynd_arr(self):
         # Positionate at the beginning of the file
         with open_file(self.path, self.mode, self.has_header) as csvfile:
-            return nd.array(csv.reader(csvfile), dtype=self.schema)
+            return nd.array(csv.reader(csvfile, **self.dialect), dtype=self.schema)
+
 
     def __array__(self):
         return nd.as_numpy(self.dynd_arr())
@@ -143,7 +155,8 @@ def __getitem__(self, key):
                 start, stop, step = key.start, key.stop, key.step
             else:
                 raise IndexError("key '%r' is not valid" % key)
-            read_iter = it.islice(csv.reader(csvfile), start, stop, step)
+            read_iter = it.islice(csv.reader(csvfile, **self.dialect),
+                                  start, stop, step)
             res = nd.array(read_iter, dtype=self.schema)
         return DyND_DDesc(res)
 
@@ -153,14 +166,14 @@ def __setitem__(self, key, value):
 
     def __iter__(self):
         return csv_descriptor_iter(
-            self.path, self.mode, self.has_header, self.schema)
+            self.path, self.mode, self.has_header, self.schema, self.dialect)
 
     def append(self, row):
         """Append a row of values (in sequence form)."""
         values = nd.array(row, dtype=self.schema)  # validate row
         with open_file(self.path, self.mode, self.has_header) as csvfile:
             csvfile.seek(0, os.SEEK_END)  # go to the end of the file
-            delimiter = self.dialect.delimiter
+            delimiter = self.dialect['delimiter']
             csvfile.write(delimiter.join(py2help.unicode(v) for v in row)+'\n')
 
     def iterchunks(self, blen=None, start=None, stop=None):
@@ -185,7 +198,7 @@ def iterchunks(self, blen=None, start=None, stop=None):
         # Return the iterable
         return csv_descriptor_iterchunks(
             self.path, self.mode, self.has_header,
-            self.schema, blen, start, stop)
+            self.schema, blen, self.dialect, start, stop)
 
     def remove(self):
         """Remove the persistent storage."""


Test patch:
diff --git a/blaze/datadescriptor/tests/test_csv_data_descriptor.py b/blaze/datadescriptor/tests/test_csv_data_descriptor.py
--- a/blaze/datadescriptor/tests/test_csv_data_descriptor.py
+++ b/blaze/datadescriptor/tests/test_csv_data_descriptor.py
@@ -3,33 +3,71 @@
 import unittest
 import tempfile
 import os
+import csv
 
 import datashape
 
 from blaze.datadescriptor import (
     CSV_DDesc, DyND_DDesc, DDesc, ddesc_as_py)
 
-# A CSV toy example
-csv_buf = u"""k1,v1,1,False
-k2,v2,2,True
-k3,v3,3,False
-"""
-csv_schema = "{ f0: string, f1: string, f2: int16, f3: bool }"
+
+def sanitize(lines):
+    return '\n'.join(line.strip() for line in lines.split('\n'))
+
+
+class TestCSV_DDesc_dialect(unittest.TestCase):
+
+    buf = sanitize(
+    u"""Name Amount
+        Alice 100
+        Bob 200
+        Alice 50
+    """)
+
+    schema = "{ f0: string, f1: int }"
+
+    def setUp(self):
+        handle, self.csv_file = tempfile.mkstemp(".csv")
+        with os.fdopen(handle, "w") as f:
+            f.write(self.buf)
+
+    def tearDown(self):
+        os.remove(self.csv_file)
+
+    def test_overwrite_delimiter(self):
+        dd = CSV_DDesc(self.csv_file, dialect='excel', schema=self.schema,
+                delimiter=' ')
+        assert dd.dialect['delimiter'] == ' '
+
+    def test_content(self):
+        dd = CSV_DDesc(self.csv_file, dialect='excel', schema=self.schema,
+                delimiter=' ')
+        print(ddesc_as_py(dd))
+        s = str(ddesc_as_py(dd))
+        assert 'Alice' in s and 'Bob' in s
 
 
 class TestCSV_DDesc(unittest.TestCase):
 
+    # A CSV toy example
+    buf = sanitize(
+    u"""k1,v1,1,False
+        k2,v2,2,True
+        k3,v3,3,False
+    """)
+    schema = "{ f0: string, f1: string, f2: int16, f3: bool }"
+
     def setUp(self):
         handle, self.csv_file = tempfile.mkstemp(".csv")
         with os.fdopen(handle, "w") as f:
-            f.write(csv_buf)
+            f.write(self.buf)
 
     def tearDown(self):
         os.remove(self.csv_file)
 
     def test_basic_object_type(self):
         self.assertTrue(issubclass(CSV_DDesc, DDesc))
-        dd = CSV_DDesc(self.csv_file, schema=csv_schema)
+        dd = CSV_DDesc(self.csv_file, schema=self.schema)
         self.assertTrue(isinstance(dd, DDesc))
         self.assertTrue(isinstance(dd.dshape.shape[0], datashape.Var))
         self.assertEqual(ddesc_as_py(dd), [
@@ -38,7 +76,7 @@ def test_basic_object_type(self):
             {u'f0': u'k3', u'f1': u'v3', u'f2': 3, u'f3': False}])
 
     def test_iter(self):
-        dd = CSV_DDesc(self.csv_file, schema=csv_schema)
+        dd = CSV_DDesc(self.csv_file, schema=self.schema)
 
         # Iteration should produce DyND_DDesc instances
         vals = []
@@ -52,7 +90,7 @@ def test_iter(self):
             {u'f0': u'k3', u'f1': u'v3', u'f2': 3, u'f3': False}])
 
     def test_iterchunks(self):
-        dd = CSV_DDesc(self.csv_file, schema=csv_schema)
+        dd = CSV_DDesc(self.csv_file, schema=self.schema)
 
         # Iteration should produce DyND_DDesc instances
         vals = []
@@ -66,7 +104,7 @@ def test_iterchunks(self):
             {u'f0': u'k3', u'f1': u'v3', u'f2': 3, u'f3': False}])
 
     def test_iterchunks_start(self):
-        dd = CSV_DDesc(self.csv_file, schema=csv_schema)
+        dd = CSV_DDesc(self.csv_file, schema=self.schema)
         vals = []
         for el in dd.iterchunks(blen=2, start=1):
             vals.extend(ddesc_as_py(el))
@@ -75,14 +113,14 @@ def test_iterchunks_start(self):
             {u'f0': u'k3', u'f1': u'v3', u'f2': 3, u'f3': False}])
 
     def test_iterchunks_stop(self):
-        dd = CSV_DDesc(self.csv_file, schema=csv_schema)
+        dd = CSV_DDesc(self.csv_file, schema=self.schema)
         vals = [ddesc_as_py(v) for v in dd.iterchunks(blen=1, stop=2)]
         self.assertEqual(vals, [
             [{u'f0': u'k1', u'f1': u'v1', u'f2': 1, u'f3': False}],
             [{u'f0': u'k2', u'f1': u'v2', u'f2': 2, u'f3': True}]])
 
     def test_iterchunks_start_stop(self):
-        dd = CSV_DDesc(self.csv_file, schema=csv_schema)
+        dd = CSV_DDesc(self.csv_file, schema=self.schema)
         vals = [ddesc_as_py(v) for v in dd.iterchunks(blen=1, start=1, stop=2)]
         self.assertEqual(vals, [[
             {u'f0': u'k2', u'f1': u'v2', u'f2': 2, u'f3': True}]])
@@ -91,8 +129,8 @@ def test_append(self):
         # Get a private file so as to not mess the original one
         handle, csv_file = tempfile.mkstemp(".csv")
         with os.fdopen(handle, "w") as f:
-            f.write(csv_buf)
-        dd = CSV_DDesc(csv_file, schema=csv_schema, mode='r+')
+            f.write(self.buf)
+        dd = CSV_DDesc(csv_file, schema=self.schema, mode='r+')
         dd.append(["k4", "v4", 4, True])
         vals = [ddesc_as_py(v) for v in dd.iterchunks(blen=1, start=3)]
         self.assertEqual(vals, [[
@@ -100,7 +138,7 @@ def test_append(self):
         os.remove(csv_file)
 
     def test_getitem_start(self):
-        dd = CSV_DDesc(self.csv_file, schema=csv_schema)
+        dd = CSV_DDesc(self.csv_file, schema=self.schema)
         el = dd[0]
         self.assertTrue(isinstance(el, DyND_DDesc))
         vals = ddesc_as_py(el)
@@ -108,7 +146,7 @@ def test_getitem_start(self):
             {u'f0': u'k1', u'f1': u'v1', u'f2': 1, u'f3': False}])
 
     def test_getitem_stop(self):
-        dd = CSV_DDesc(self.csv_file, schema=csv_schema)
+        dd = CSV_DDesc(self.csv_file, schema=self.schema)
         el = dd[:1]
         self.assertTrue(isinstance(el, DyND_DDesc))
         vals = ddesc_as_py(el)
@@ -116,7 +154,7 @@ def test_getitem_stop(self):
             {u'f0': u'k1', u'f1': u'v1', u'f2': 1, u'f3': False}])
 
     def test_getitem_step(self):
-        dd = CSV_DDesc(self.csv_file, schema=csv_schema)
+        dd = CSV_DDesc(self.csv_file, schema=self.schema)
         el = dd[::2]
         self.assertTrue(isinstance(el, DyND_DDesc))
         vals = ddesc_as_py(el)
@@ -125,7 +163,7 @@ def test_getitem_step(self):
             {u'f0': u'k3', u'f1': u'v3', u'f2': 3, u'f3': False}])
 
     def test_getitem_start_step(self):
-        dd = CSV_DDesc(self.csv_file, schema=csv_schema)
+        dd = CSV_DDesc(self.csv_file, schema=self.schema)
         el = dd[1::2]
         self.assertTrue(isinstance(el, DyND_DDesc))
         vals = ddesc_as_py(el)