Repository: frictionlessdata/tabulator-py
Pull request #54 (2016-05-05 10:42:12)

Linear TSV format support
See: http://dataprotocols.org/linear-tsv/

Depends on https://github.com/solidsnack/tsv/pull/2 for Python 3 support.

coveralls: 
[![Coverage Status](https://coveralls.io/builds/6059777/badge)](https://coveralls.io/builds/6059777)

Coverage decreased (-1.3%) to 86.853% when pulling **47ca52c7c02d5d1c4a2bb1d2e3af6ae5ecae04ca on sirex:master** into **b9e4553ab2088e1f03a04af6fa7cf846b767b245 on frictionlessdata:master**.

coveralls: 
[![Coverage Status](https://coveralls.io/builds/6097804/badge)](https://coveralls.io/builds/6097804)

Coverage decreased (-1.2%) to 86.985% when pulling **e92a8fd773fa67db35cc3a4bdca71139797e5bd6 on sirex:master** into **b9e4553ab2088e1f03a04af6fa7cf846b767b245 on frictionlessdata:master**.


Patch:
diff --git a/data/table.tsv b/data/table.tsv
new file mode 100644
index 0000000..fed6113
--- /dev/null
+++ b/data/table.tsv
@@ -0,0 +1,4 @@
+id	name
+1	english
+2	中国人
+3	\N
diff --git a/examples/topen.py b/examples/topen.py
index 2e1513a..2fd96b8 100644
--- a/examples/topen.py
+++ b/examples/topen.py
@@ -19,6 +19,14 @@ with topen(source, with_headers=True) as table:
         print(row)
 
 
+print('\nParse linear tsv format:')
+source = 'data/table.tsv'
+with topen(source, with_headers=True) as table:
+    print(table.headers)
+    for row in table:
+        print(row)
+
+
 print('\nParse json with dicts:')
 source = 'file://data/table-dicts.json'
 with topen(source, with_headers=True) as table:
diff --git a/setup.py b/setup.py
index 55b1335..dbaa1eb 100644
--- a/setup.py
+++ b/setup.py
@@ -27,6 +27,7 @@ INSTALL_REQUIRES = [
     'openpyxl>=2.0',
     'requests>=2.8',
     'jsontableschema>=0.5',
+    'linear-tsv>=0.99.1',
 ]
 LINT_REQUIRES = [
     'pylint',
diff --git a/tabulator/VERSION b/tabulator/VERSION
index e473765..0b69c00 100644
--- a/tabulator/VERSION
+++ b/tabulator/VERSION
@@ -1,1 +1,1 @@
-0.3.13
+0.3.14
diff --git a/tabulator/helpers.py b/tabulator/helpers.py
index e8e3170..ae1c807 100644
--- a/tabulator/helpers.py
+++ b/tabulator/helpers.py
@@ -11,6 +11,8 @@ from six.moves.urllib.parse import urlparse
 
 from . import errors
 
+CHARSET_DETECTION_MAX_LINES = 1000
+CHARSET_DETECTION_MIN_CONFIDENCE = 0.5
 
 # Module API
 
@@ -54,18 +56,21 @@ def detect_encoding(bytes):
     """Detect encoding of a byte stream.
     """
     detector = UniversalDetector()
-    for line in bytes.readlines():
+    num_lines = CHARSET_DETECTION_MAX_LINES
+    while num_lines > 0:
+        line = bytes.readline()
         detector.feed(line)
         if detector.done:
             # TODO: does it work?
             break
+        num_lines -= 1
     detector.close()
     bytes.seek(0)
     confidence = detector.result['confidence']
     encoding = detector.result['encoding']
     # Do not use if not confident
-    if confidence < 0.95:
-        encoding = None
+    if confidence < CHARSET_DETECTION_MIN_CONFIDENCE:
+        encoding = 'utf-8'
     # Default to utf-8 for safety
     if encoding == 'ascii':
         encoding = 'utf-8'
diff --git a/tabulator/parsers/__init__.py b/tabulator/parsers/__init__.py
index c184319..c33e0b0 100644
--- a/tabulator/parsers/__init__.py
+++ b/tabulator/parsers/__init__.py
@@ -6,6 +6,7 @@ from __future__ import unicode_literals
 
 from .api import Parser as API
 from .csv import CSVParser as CSV
+from .tsv import TSVParser as TSV
 from .excel import ExcelParser as Excel
 from .excelx import ExcelxParser as Excelx
 from .json import JSONParser as JSON
diff --git a/tabulator/parsers/tsv.py b/tabulator/parsers/tsv.py
new file mode 100644
index 0000000..a438ca1
--- /dev/null
+++ b/tabulator/parsers/tsv.py
@@ -0,0 +1,56 @@
+# -*- coding: utf-8 -*-
+from __future__ import division
+from __future__ import print_function
+from __future__ import absolute_import
+from __future__ import unicode_literals
+
+import tsv
+
+from .. import helpers
+from . import api
+
+
+# Module API
+
+class TSVParser(api.Parser):
+    """Parser to parse linear TSV data format.
+
+    See: http://dataprotocols.org/linear-tsv/
+    """
+
+    # Public
+
+    def __init__(self, **options):
+        self.__options = options
+        self.__loader = None
+        self.__chars = None
+        self.__items = None
+
+    def open(self, loader):
+        self.close()
+        self.__loader = loader
+        self.__chars = loader.load(mode='t')
+        self.reset()
+
+    def close(self):
+        if not self.closed:
+            self.__chars.close()
+
+    @property
+    def closed(self):
+        return self.__chars is None or self.__chars.closed
+
+    @property
+    def items(self):
+        return self.__items
+
+    def reset(self):
+        helpers.reset_stream(self.__chars)
+        self.__items = self.__emit_items()
+
+    # Private
+
+    def __emit_items(self):
+        items = tsv.un(self.__chars)
+        for item in items:
+            yield (None, tuple(item))
diff --git a/tabulator/topen.py b/tabulator/topen.py
index 3389b5e..db63cfa 100644
--- a/tabulator/topen.py
+++ b/tabulator/topen.py
@@ -46,6 +46,7 @@ def topen(source,
         Format of source:
             - None (detect)
             - csv
+            - tsv
             - json
             - xls
             - xlsx
@@ -138,6 +139,7 @@ _LOADERS = {
 
 _PARSERS = {
     'csv': parsers.CSV,
+    'tsv': parsers.TSV,
     'xls': parsers.Excel,
     'xlsx': parsers.Excelx,
     'json': parsers.JSON,


Test patch:
diff --git a/tests/parsers/test_csv.py b/tests/parsers/test_csv.py
index 55fe1ba..de154e8 100644
--- a/tests/parsers/test_csv.py
+++ b/tests/parsers/test_csv.py
@@ -20,7 +20,7 @@ class TestCSV(unittest.TestCase):
         basedir = os.path.join(os.path.dirname(__file__), '..', '..')
         self.source = os.path.join(basedir, 'data', 'table.csv')
         self.loader = Mock()
-        self.loader.load = Mock(return_value=io.open(self.source))
+        self.loader.load = Mock(return_value=io.open(self.source, encoding='utf-8'))
         self.parser = parsers.CSV()
 
     # Tests
diff --git a/tests/parsers/test_tsv.py b/tests/parsers/test_tsv.py
new file mode 100644
index 0000000..495dec0
--- /dev/null
+++ b/tests/parsers/test_tsv.py
@@ -0,0 +1,46 @@
+# -*- coding: utf-8 -*-
+from __future__ import division
+from __future__ import print_function
+from __future__ import absolute_import
+from __future__ import unicode_literals
+
+import io
+import os
+import unittest
+from mock import Mock
+
+from tabulator import parsers
+
+
+class TestTSV(unittest.TestCase):
+
+    # Actions
+
+    def setUp(self):
+        basedir = os.path.join(os.path.dirname(__file__), '..', '..')
+        self.source = os.path.join(basedir, 'data', 'table.tsv')
+        self.loader = Mock()
+        self.loader.load = Mock(return_value=io.open(self.source))
+        self.parser = parsers.TSV()
+
+    # Tests
+
+    def test(self):
+
+        self.assertTrue(self.parser.closed)
+        self.parser.open(self.loader)
+        self.assertFalse(self.parser.closed)
+
+        self.assertEqual(list(self.parser.items), [
+            (None, ('id', 'name')),
+            (None, ('1', 'english')),
+            (None, ('2', '中国人')),
+            (None, ('3', None)),
+        ])
+
+        self.assertEqual(len(list(self.parser.items)), 0)
+        self.parser.reset()
+        self.assertEqual(len(list(self.parser.items)), 4)
+
+        self.parser.close()
+        self.assertTrue(self.parser.closed)
diff --git a/tests/test_helpers.py b/tests/test_helpers.py
index 897498d..091dbfb 100644
--- a/tests/test_helpers.py
+++ b/tests/test_helpers.py
@@ -56,6 +56,17 @@ class Test_detect_encoding(unittest.TestCase):
         bytes = io.open(path, 'rb')
         self.assertEqual(helpers.detect_encoding(bytes), 'utf-8')
 
+    def test_unknown(self):
+        bytes = io.BytesIO(b'\xff\x81')
+        self.assertEqual(helpers.detect_encoding(bytes), 'utf-8')
+
+    def test_long(self):
+        bytes = io.BytesIO(b'A\n' * 1000 + b'\xff\xff')
+        self.assertEqual(helpers.detect_encoding(bytes), 'utf-8')
+
+    def test_not_so_long(self):
+        bytes = io.BytesIO(b'A\n' * 999 + b'\xff\xff')
+        self.assertEqual(helpers.detect_encoding(bytes), 'windows-1252')
 
 class Test_reset_stream(unittest.TestCase):