Repository: frictionlessdata/tabulator-py (python)

Issue:
Replace chardet by C version (cchardet)
# Overview

To improve detection speed we better use - https://github.com/PyYoshi/cChardet

Pull request:
Rebased on cchardet to improve speed
- fixes #104 

Patch:
diff --git a/pylama.ini b/pylama.ini
index 362a793..08c5f40 100644
--- a/pylama.ini
+++ b/pylama.ini
@@ -1,6 +1,6 @@
 [pylama]
 linters = pyflakes,mccabe,pep8
-ignore = E731
+ignore = E301,E731
 
 [pylama:mccabe]
 complexity = 16
diff --git a/setup.py b/setup.py
index 34ccf0f..103ccdb 100644
--- a/setup.py
+++ b/setup.py
@@ -23,14 +23,14 @@ INSTALL_REQUIRES = [
     'six>=1.9,<2.0a',
     'xlrd>=1.0,<2.0a',
     'ijson>=2.0,<3.0a',
-    'chardet>=2.0,<3.0a',
+    'cchardet>=1.0,<2.0a',
     'openpyxl>=2.4,<3.0a',
     'requests>=2.8,<3.0a',
     'linear-tsv>=1.0,<2.0a',
     'unicodecsv>=0.14,<1.0a',
-    'jsonlines>=1.1,<1.2',
-    'ezodf>=0.3,<0.4',
-    'lxml',  # required by ezodf
+    'jsonlines>=1.1,<2.0a',
+    'ezodf>=0.3,<1.0a',
+    'lxml>=3.0,<4.0a',  # required by ezodf
 ]
 TESTS_REQUIRE = [
     'pylama',
diff --git a/tabulator/config.py b/tabulator/config.py
index f6e4585..1193ba6 100644
--- a/tabulator/config.py
+++ b/tabulator/config.py
@@ -9,8 +9,8 @@ from __future__ import unicode_literals
 
 DEFAULT_SCHEME = 'file'
 DEFAULT_ENCODING = 'utf-8'
-ENCODING_DETECTION_MAX_LINES = 1000
-ENCODING_DETECTION_MIN_CONFIDENCE = 0.5
+BYTES_SAMPLE_SIZE = 1000
+ENCODING_CONFIDENCE = 0.5
 LOADERS = {
     'file': 'tabulator.loaders.file.FileLoader',
     'ftp': 'tabulator.loaders.web.WebLoader',
diff --git a/tabulator/helpers.py b/tabulator/helpers.py
index 4f9f5f6..ca99d11 100644
--- a/tabulator/helpers.py
+++ b/tabulator/helpers.py
@@ -56,46 +56,35 @@ def detect_format(source):
     return format
 
 
-def detect_encoding(bytes, encoding=None):
-    """Detect encoding of a byte stream.
+def detect_encoding(sample, encoding=None):
+    """Detect encoding of a byte string sample.
     """
     # To reduce tabulator import time
-    from chardet.universaldetector import UniversalDetector
-    if encoding is not None:
-        if encoding.lower() == 'utf-8':
-            prefix = bytes.read(len(codecs.BOM_UTF8))
-            if prefix == codecs.BOM_UTF8:
+    from cchardet import detect
+    def detect_utf8_sig(sample, encoding):
+        if encoding == 'utf-8':
+            if sample.startswith(codecs.BOM_UTF8):
                 encoding = 'utf-8-sig'
-            bytes.seek(0)
         return encoding
-    detector = UniversalDetector()
-    num_lines = config.ENCODING_DETECTION_MAX_LINES
-    while num_lines > 0:
-        line = bytes.readline()
-        detector.feed(line)
-        if detector.done:
-            break
-        num_lines -= 1
-    detector.close()
-    bytes.seek(0)
-    confidence = detector.result['confidence']
-    encoding = detector.result['encoding']
-    # Do not use if not confident
-    if confidence < config.ENCODING_DETECTION_MIN_CONFIDENCE:
+    if encoding is not None:
+        encoding = encoding.lower()
+        return detect_utf8_sig(sample, encoding)
+    result = detect(sample)
+    confidence = result['confidence'] or 0
+    encoding = (result['encoding'] or '').lower()
+    encoding = detect_utf8_sig(sample, encoding)
+    if confidence < config.ENCODING_CONFIDENCE:
         encoding = config.DEFAULT_ENCODING
-    # Default to utf-8 for safety
     if encoding == 'ascii':
         encoding = config.DEFAULT_ENCODING
     return encoding
 
 
-detect_html_re = re.compile('\s*<(!doctype|html)', re.IGNORECASE)
-
-
 def detect_html(text):
     """Detect if text is HTML.
     """
-    return bool(detect_html_re.match(text))
+    pattern = re.compile('\s*<(!doctype|html)', re.IGNORECASE)
+    return bool(pattern.match(text))
 
 
 def reset_stream(stream):
diff --git a/tabulator/loaders/file.py b/tabulator/loaders/file.py
index 1acb100..e312bf7 100644
--- a/tabulator/loaders/file.py
+++ b/tabulator/loaders/file.py
@@ -7,6 +7,7 @@ from __future__ import unicode_literals
 import io
 from .. import exceptions
 from .. import helpers
+from .. import config
 from . import api
 
 
@@ -30,11 +31,13 @@ class FileLoader(api.Loader):
         # Prepare bytes
         try:
             bytes = io.open(source, 'rb')
+            sample = bytes.read(config.BYTES_SAMPLE_SIZE)
+            bytes.seek(0)
         except IOError as exception:
             raise exceptions.IOError(str(exception))
 
         # Prepare encoding
-        encoding = helpers.detect_encoding(bytes, encoding)
+        encoding = helpers.detect_encoding(sample, encoding)
 
         # Return or raise
         if mode == 'b':
diff --git a/tabulator/loaders/stream.py b/tabulator/loaders/stream.py
index 99ee412..d23c600 100644
--- a/tabulator/loaders/stream.py
+++ b/tabulator/loaders/stream.py
@@ -7,6 +7,7 @@ from __future__ import unicode_literals
 import io
 from .. import exceptions
 from .. import helpers
+from .. import config
 from . import api
 
 
@@ -29,9 +30,11 @@ class StreamLoader(api.Loader):
 
         # Prepare bytes
         bytes = source
+        sample = bytes.read(config.BYTES_SAMPLE_SIZE)
+        bytes.seek(0)
 
         # Prepare encoding
-        encoding = helpers.detect_encoding(bytes, encoding)
+        encoding = helpers.detect_encoding(sample, encoding)
 
         # Return or raise
         if mode == 'b':
diff --git a/tabulator/loaders/web.py b/tabulator/loaders/web.py
index 2b63c05..3f6f327 100644
--- a/tabulator/loaders/web.py
+++ b/tabulator/loaders/web.py
@@ -10,6 +10,7 @@ from six.moves.urllib.error import URLError
 from six.moves.urllib.request import urlopen
 from .. import exceptions
 from .. import helpers
+from .. import config
 from . import api
 
 
@@ -38,6 +39,8 @@ class WebLoader(api.Loader):
             else:
                 bytes = _WebStream(source)
                 response = bytes.response
+            sample = bytes.read(config.BYTES_SAMPLE_SIZE)
+            bytes.seek(0)
         except URLError as exception:
             raise exceptions.HTTPError(str(exception))
 
@@ -47,7 +50,7 @@ class WebLoader(api.Loader):
                 encoding = response.headers.getparam('charset')
             else:
                 encoding = response.headers.get_content_charset()
-        encoding = helpers.detect_encoding(bytes, encoding)
+        encoding = helpers.detect_encoding(sample, encoding)
 
         # Return or raise
         if mode == 'b':
@@ -65,7 +68,7 @@ class _WebStream(object):
 
     def __init__(self, source):
         self.__source = source
-        self.__response = self.__make_request()
+        self.__response = urlopen(self.__source)
 
     def __getattr__(self, name):
         return getattr(self.__response, name)
@@ -79,9 +82,4 @@ class _WebStream(object):
 
     def seek(self, offset):
         assert offset == 0
-        self.__response = self.__make_request()
-
-    # Private
-
-    def __make_request(self):
-        return urlopen(self.__source)
+        self.__response = urlopen(self.__source)


Test patch:
diff --git a/tests/test_helpers.py b/tests/test_helpers.py
index bca3de2..40e4144 100644
--- a/tests/test_helpers.py
+++ b/tests/test_helpers.py
@@ -6,7 +6,7 @@ from __future__ import unicode_literals
 
 import io
 import pytest
-from tabulator import helpers
+from tabulator import helpers, config
 
 
 # Tests
@@ -41,23 +41,18 @@ def test_detect_format_works_with_urls_with_query_and_fragment_components():
 
 
 def test_detect_encoding():
-    bytes = io.open('README.md', 'rb')
-    assert helpers.detect_encoding(bytes) == 'utf-8'
+    sample = io.open('README.md', 'rb').read(config.BYTES_SAMPLE_SIZE)
+    assert helpers.detect_encoding(sample) == 'utf-8'
 
 
-def test_detect_encoding_unknown():
-    bytes = io.BytesIO(b'\xff\x81')
-    assert helpers.detect_encoding(bytes) == 'utf-8'
-
+def test_detect_encoding_windows_1252():
+    sample = b'A\n' * 300 + b'\xff\xff'
+    assert helpers.detect_encoding(sample) == 'windows-1252'
 
-def test_detect_encoding_long():
-    bytes = io.BytesIO(b'A\n' * 1000 + b'\xff\xff')
-    assert helpers.detect_encoding(bytes) == 'utf-8'
 
-
-def test_detect_encoding_not_so_long():
-    bytes = io.BytesIO(b'A\n' * 999 + b'\xff\xff')
-    assert helpers.detect_encoding(bytes) == 'windows-1252'
+def test_detect_encoding_unknown():
+    sample = b'\xff\x81'
+    assert helpers.detect_encoding(sample) == 'utf-8'
 
 
 def test_reset_stream_seekable():