Repository: frictionlessdata/tabulator-py (python)

Issue:
CSV dialect guessing?
# Overview

For csv options not passed explicitly `tabulator` should use python default csv sniffer.  

Pull request:
Added csv dialect guessing
- fixes #79

Patch:
diff --git a/tabulator/loaders/file.py b/tabulator/loaders/file.py
index 7d5ecaf..f085b2d 100644
--- a/tabulator/loaders/file.py
+++ b/tabulator/loaders/file.py
@@ -36,8 +36,8 @@ class FileLoader(api.Loader):
         except IOError as exception:
             raise exceptions.IOError(str(exception))
         if helpers.detect_zip(sample):
-            message = 'Source has been detected as ZIP (not supported)'
-            raise exceptions.SourceError(message)
+            message = 'Format has been detected as ZIP (not supported)'
+            raise exceptions.FormatError(message)
 
         # Prepare encoding
         encoding = helpers.detect_encoding(sample, encoding)
diff --git a/tabulator/loaders/stream.py b/tabulator/loaders/stream.py
index 5bbad46..e94d1a0 100644
--- a/tabulator/loaders/stream.py
+++ b/tabulator/loaders/stream.py
@@ -33,8 +33,8 @@ class StreamLoader(api.Loader):
         sample = bytes.read(config.BYTES_SAMPLE_SIZE)
         bytes.seek(0)
         if helpers.detect_zip(sample):
-            message = 'Source has been detected as ZIP (not supported)'
-            raise exceptions.SourceError(message)
+            message = 'Format has been detected as ZIP (not supported)'
+            raise exceptions.FormatError(message)
 
         # Prepare encoding
         encoding = helpers.detect_encoding(sample, encoding)
diff --git a/tabulator/loaders/web.py b/tabulator/loaders/web.py
index d3281b6..ac5b32d 100644
--- a/tabulator/loaders/web.py
+++ b/tabulator/loaders/web.py
@@ -44,8 +44,8 @@ class WebLoader(api.Loader):
         except URLError as exception:
             raise exceptions.HTTPError(str(exception))
         if helpers.detect_zip(sample):
-            message = 'Source has been detected as ZIP (not supported)'
-            raise exceptions.SourceError(message)
+            message = 'Format has been detected as ZIP (not supported)'
+            raise exceptions.FormatError(message)
 
         # Prepare encoding
         if encoding is None:
diff --git a/tabulator/parsers/csv.py b/tabulator/parsers/csv.py
index 7413d4b..2a2b73a 100644
--- a/tabulator/parsers/csv.py
+++ b/tabulator/parsers/csv.py
@@ -6,6 +6,7 @@ from __future__ import unicode_literals
 
 import csv
 import six
+from itertools import chain
 from codecs import iterencode
 from .. import helpers
 from . import api
@@ -72,7 +73,9 @@ class CSVParser(api.Parser):
         if six.PY2:
             # Reader requires utf-8 encoded stream
             bytes = iterencode(self.__chars, 'utf-8')
-            items = csv.reader(bytes, **self.__options)
+            sample = next(bytes)
+            dialect = self.__prepare_dialect(sample)
+            items = csv.reader(chain([sample], bytes), dialect=dialect)
             for number, item in enumerate(items, start=1):
                 values = []
                 for value in item:
@@ -82,6 +85,17 @@ class CSVParser(api.Parser):
 
         # For PY3 use chars
         else:
-            items = csv.reader(self.__chars, **self.__options)
+            sample = next(self.__chars)
+            dialect = self.__prepare_dialect(sample)
+            items = csv.reader(chain([sample], self.__chars), dialect=dialect)
             for number, item in enumerate(items, start=1):
                 yield (number, None, list(item))
+
+    def __prepare_dialect(self, sample):
+        try:
+            dialect = csv.Sniffer().sniff(sample)
+        except csv.Error:
+            dialect = csv.excel
+        for key, value in self.__options.items():
+            setattr(dialect, key, value)
+        return dialect
diff --git a/tabulator/stream.py b/tabulator/stream.py
index 65cb9c4..616ee6f 100644
--- a/tabulator/stream.py
+++ b/tabulator/stream.py
@@ -342,8 +342,8 @@ class Stream(object):
                     text += value
         html_source = helpers.detect_html(text)
         if html_source:
-            message = 'Source has been detected as HTML (not supported)'
-            raise exceptions.SourceError(message)
+            message = 'Format has been detected as HTML (not supported)'
+            raise exceptions.FormatError(message)
 
     def __apply_processors(self, iterator):
 


Test patch:
diff --git a/tests/test_stream.py b/tests/test_stream.py
index 3b0fe62..79f66bd 100644
--- a/tests/test_stream.py
+++ b/tests/test_stream.py
@@ -8,10 +8,30 @@ import pytest
 from tabulator import Stream, exceptions
 
 
+# Tests [formats]
+
+def test_stream_csv_excel():
+    source = 'value1,value2\nvalue3,value4'
+    with Stream(source, scheme='text', format='csv') as stream:
+        assert stream.read() == [['value1', 'value2'], ['value3', 'value4']]
+
+
+def test_stream_csv_excel_tab():
+    source = 'value1\tvalue2\nvalue3\tvalue4'
+    with Stream(source, scheme='text', format='csv') as stream:
+        assert stream.read() == [['value1', 'value2'], ['value3', 'value4']]
+
+
+def test_stream_csv_unix():
+    source = '"value1","value2"\n"value3","value4"'
+    with Stream(source, scheme='text', format='csv') as stream:
+        assert stream.read() == [['value1', 'value2'], ['value3', 'value4']]
+
+
 # Tests [options]
 
 def test_stream_csv_delimiter():
-    source = 'value1;value2\nvalue3;value4'
+    source = '"value1";"value2"\n"value3";"value4"'
     with Stream(source, scheme='text', format='csv', delimiter=';') as stream:
         assert stream.read() == [['value1', 'value2'], ['value3', 'value4']]
 
@@ -54,23 +74,23 @@ def test_stream_json_prefix():
 
 # Tests [errors]
 
-def test_stream_source_error_zip():
-    stream = Stream('data/special/table.csv.zip', format='csv')
+def test_stream_source_error_data():
+    stream = Stream('[1,2]', scheme='text', format='json')
     with pytest.raises(exceptions.SourceError) as excinfo:
         stream.open()
+        stream.read()
 
 
-def test_stream_source_error_html():
-    stream = Stream('data/special/table.csv.html', format='csv')
-    with pytest.raises(exceptions.SourceError) as excinfo:
+def test_stream_format_error_zip():
+    stream = Stream('data/special/table.csv.zip', format='csv')
+    with pytest.raises(exceptions.FormatError) as excinfo:
         stream.open()
 
 
-def test_stream_source_error_data():
-    stream = Stream('[1,2]', scheme='text', format='json')
-    with pytest.raises(exceptions.SourceError) as excinfo:
+def test_stream_format_error_html():
+    stream = Stream('data/special/table.csv.html', format='csv')
+    with pytest.raises(exceptions.FormatError) as excinfo:
         stream.open()
-        stream.read()
 
 
 def test_stream_scheme_error():
diff --git a/tests/test_topen.py b/tests/test_topen.py
index 4901ad5..e23037d 100644
--- a/tests/test_topen.py
+++ b/tests/test_topen.py
@@ -407,7 +407,7 @@ def test_html_content():
 
     # Check raises
     source = 'https://github.com/frictionlessdata/tabulator-py/blob/master/data/table.csv'
-    with pytest.raises(exceptions.SourceError) as excinfo:
+    with pytest.raises(exceptions.FormatError) as excinfo:
         table = topen(source, headers='row1')
     assert 'HTML' in str(excinfo.value)