Repository: frictionlessdata/tabulator-py
Pull request #127 (2016-11-03 08:29:59)

Improved csv sniffing precision
- fixes https://github.com/frictionlessdata/goodtables-py/issues/114
- fixes https://github.com/frictionlessdata/goodtables-py/issues/142


roll: @kravets-levko 
please review

One thing - you still should pass `escapechar='\'` because csv sniffer can't detect it.
roll: @amercader 
It's blocking please review respectively if needed.

Patch:
diff --git a/data/special/escaping.csv b/data/special/escaping.csv
new file mode 100644
index 0000000..bfaf78b
--- /dev/null
+++ b/data/special/escaping.csv
@@ -0,0 +1,4 @@
+ID, Test
+1, "Test line 1"
+2, "Test "" line 2"
+3, "Test \" line 3"
diff --git a/tabulator/config.py b/tabulator/config.py
index 1193ba6..81ac008 100644
--- a/tabulator/config.py
+++ b/tabulator/config.py
@@ -11,6 +11,7 @@ DEFAULT_SCHEME = 'file'
 DEFAULT_ENCODING = 'utf-8'
 BYTES_SAMPLE_SIZE = 1000
 ENCODING_CONFIDENCE = 0.5
+CSV_SAMPLE_LINES = 100
 LOADERS = {
     'file': 'tabulator.loaders.file.FileLoader',
     'ftp': 'tabulator.loaders.web.WebLoader',
diff --git a/tabulator/parsers/csv.py b/tabulator/parsers/csv.py
index 2a2b73a..2c63e3d 100644
--- a/tabulator/parsers/csv.py
+++ b/tabulator/parsers/csv.py
@@ -9,6 +9,7 @@ import six
 from itertools import chain
 from codecs import iterencode
 from .. import helpers
+from .. import config
 from . import api
 
 
@@ -73,9 +74,8 @@ class CSVParser(api.Parser):
         if six.PY2:
             # Reader requires utf-8 encoded stream
             bytes = iterencode(self.__chars, 'utf-8')
-            sample = next(bytes)
-            dialect = self.__prepare_dialect(sample)
-            items = csv.reader(chain([sample], bytes), dialect=dialect)
+            sample, dialect = self.__prepare_dialect(bytes)
+            items = csv.reader(chain(sample, bytes), dialect=dialect)
             for number, item in enumerate(items, start=1):
                 values = []
                 for value in item:
@@ -85,17 +85,31 @@ class CSVParser(api.Parser):
 
         # For PY3 use chars
         else:
-            sample = next(self.__chars)
-            dialect = self.__prepare_dialect(sample)
-            items = csv.reader(chain([sample], self.__chars), dialect=dialect)
+            sample, dialect = self.__prepare_dialect(self.__chars)
+            items = csv.reader(chain(sample, self.__chars), dialect=dialect)
             for number, item in enumerate(items, start=1):
                 yield (number, None, list(item))
 
-    def __prepare_dialect(self, sample):
+    def __prepare_dialect(self, stream):
+
+        # Get sample
+        sample = []
+        while True:
+            try:
+                sample.append(next(stream))
+            except StopIteration:
+                break
+            if len(sample) >= config.CSV_SAMPLE_LINES:
+                break
+
+        # Get dialect
         try:
-            dialect = csv.Sniffer().sniff(sample)
+            separator = b'' if six.PY2 else ''
+            delimiter = self.__options.get('delimiter', ',')
+            dialect = csv.Sniffer().sniff(separator.join(sample), delimiter)
         except csv.Error:
             dialect = csv.excel
         for key, value in self.__options.items():
             setattr(dialect, key, value)
-        return dialect
+
+        return sample, dialect


Test patch:
diff --git a/tests/test_stream.py b/tests/test_stream.py
index 79f66bd..a5b379e 100644
--- a/tests/test_stream.py
+++ b/tests/test_stream.py
@@ -18,7 +18,7 @@ def test_stream_csv_excel():
 
 def test_stream_csv_excel_tab():
     source = 'value1\tvalue2\nvalue3\tvalue4'
-    with Stream(source, scheme='text', format='csv') as stream:
+    with Stream(source, scheme='text', format='csv', delimiter='\t') as stream:
         assert stream.read() == [['value1', 'value2'], ['value3', 'value4']]
 
 
@@ -28,6 +28,16 @@ def test_stream_csv_unix():
         assert stream.read() == [['value1', 'value2'], ['value3', 'value4']]
 
 
+def test_stream_csv_escaping():
+    with Stream('data/special/escaping.csv', escapechar='\\') as stream:
+        assert stream.read() == [
+            ['ID', 'Test'],
+            ['1', 'Test line 1'],
+            ['2', 'Test " line 2'],
+            ['3', 'Test " line 3'],
+        ]
+
+
 # Tests [options]
 
 def test_stream_csv_delimiter():