Repository: frictionlessdata/tabulator-py
Pull request #55 (2016-05-10 15:44:19)

Character set detection fixes

coveralls: 
[![Coverage Status](https://coveralls.io/builds/6120754/badge)](https://coveralls.io/builds/6120754)

Coverage increased (+0.08%) to 88.276% when pulling **98c151167057ebbf5b297d42284ed4632f24e131 on akariv:master** into **b9e4553ab2088e1f03a04af6fa7cf846b767b245 on frictionlessdata:master**.

coveralls: 
[![Coverage Status](https://coveralls.io/builds/6120939/badge)](https://coveralls.io/builds/6120939)

Coverage increased (+0.08%) to 88.276% when pulling **c8cd97b6b09cdf603d83feee95b713cddedd94e2 on akariv:master** into **b9e4553ab2088e1f03a04af6fa7cf846b767b245 on frictionlessdata:master**.

coveralls: 
[![Coverage Status](https://coveralls.io/builds/6121056/badge)](https://coveralls.io/builds/6121056)

Coverage increased (+0.08%) to 88.276% when pulling **00263bac233cbefe5fcc5906622457bd01bc4657 on akariv:master** into **b9e4553ab2088e1f03a04af6fa7cf846b767b245 on frictionlessdata:master**.


Patch:
diff --git a/tabulator/VERSION b/tabulator/VERSION
index e473765..0b69c00 100644
--- a/tabulator/VERSION
+++ b/tabulator/VERSION
@@ -1,1 +1,1 @@
-0.3.13
+0.3.14
diff --git a/tabulator/helpers.py b/tabulator/helpers.py
index e8e3170..ae1c807 100644
--- a/tabulator/helpers.py
+++ b/tabulator/helpers.py
@@ -11,6 +11,8 @@ from six.moves.urllib.parse import urlparse
 
 from . import errors
 
+CHARSET_DETECTION_MAX_LINES = 1000
+CHARSET_DETECTION_MIN_CONFIDENCE = 0.5
 
 # Module API
 
@@ -54,18 +56,21 @@ def detect_encoding(bytes):
     """Detect encoding of a byte stream.
     """
     detector = UniversalDetector()
-    for line in bytes.readlines():
+    num_lines = CHARSET_DETECTION_MAX_LINES
+    while num_lines > 0:
+        line = bytes.readline()
         detector.feed(line)
         if detector.done:
             # TODO: does it work?
             break
+        num_lines -= 1
     detector.close()
     bytes.seek(0)
     confidence = detector.result['confidence']
     encoding = detector.result['encoding']
     # Do not use if not confident
-    if confidence < 0.95:
-        encoding = None
+    if confidence < CHARSET_DETECTION_MIN_CONFIDENCE:
+        encoding = 'utf-8'
     # Default to utf-8 for safety
     if encoding == 'ascii':
         encoding = 'utf-8'


Test patch:
diff --git a/tests/parsers/test_csv.py b/tests/parsers/test_csv.py
index 55fe1ba..de154e8 100644
--- a/tests/parsers/test_csv.py
+++ b/tests/parsers/test_csv.py
@@ -20,7 +20,7 @@ class TestCSV(unittest.TestCase):
         basedir = os.path.join(os.path.dirname(__file__), '..', '..')
         self.source = os.path.join(basedir, 'data', 'table.csv')
         self.loader = Mock()
-        self.loader.load = Mock(return_value=io.open(self.source))
+        self.loader.load = Mock(return_value=io.open(self.source, encoding='utf-8'))
         self.parser = parsers.CSV()
 
     # Tests
diff --git a/tests/test_helpers.py b/tests/test_helpers.py
index 897498d..091dbfb 100644
--- a/tests/test_helpers.py
+++ b/tests/test_helpers.py
@@ -56,6 +56,17 @@ class Test_detect_encoding(unittest.TestCase):
         bytes = io.open(path, 'rb')
         self.assertEqual(helpers.detect_encoding(bytes), 'utf-8')
 
+    def test_unknown(self):
+        bytes = io.BytesIO(b'\xff\x81')
+        self.assertEqual(helpers.detect_encoding(bytes), 'utf-8')
+
+    def test_long(self):
+        bytes = io.BytesIO(b'A\n' * 1000 + b'\xff\xff')
+        self.assertEqual(helpers.detect_encoding(bytes), 'utf-8')
+
+    def test_not_so_long(self):
+        bytes = io.BytesIO(b'A\n' * 999 + b'\xff\xff')
+        self.assertEqual(helpers.detect_encoding(bytes), 'windows-1252')
 
 class Test_reset_stream(unittest.TestCase):