Repository: frictionlessdata/tabulator-py
Pull request #57 (2016-08-07 19:42:43)

Feature/native source
It fixes https://github.com/frictionlessdata/tabulator-py/issues/35

coveralls: 
[![Coverage Status](https://coveralls.io/builds/7341753/badge)](https://coveralls.io/builds/7341753)

Coverage decreased (-1.4%) to 87.395% when pulling **27c30573de723ce0abea4d7e118b8b41c4510328 on feature/native-source** into **462d0109edd9e18101e6e3e87c9bfa74ddae6558 on master**.

roll: @zhenyab 
please review
coveralls: 
[![Coverage Status](https://coveralls.io/builds/7341897/badge)](https://coveralls.io/builds/7341897)

Coverage decreased (-1.5%) to 87.275% when pulling **5dd5820b408655e77cfeb021a9302c1f52205789 on feature/native-source** into **462d0109edd9e18101e6e3e87c9bfa74ddae6558 on master**.

coveralls: 
[![Coverage Status](https://coveralls.io/builds/7347162/badge)](https://coveralls.io/builds/7347162)

Coverage decreased (-1.5%) to 87.321% when pulling **5fed8c5e381ac5bd11677252a60165a7d9095641 on feature/native-source** into **462d0109edd9e18101e6e3e87c9bfa74ddae6558 on master**.


Patch:
diff --git a/README.md b/README.md
index 9047108..be66eea 100644
--- a/README.md
+++ b/README.md
@@ -172,6 +172,11 @@ API documentation is presented as docstrings:
     - [Parser API](https://github.com/frictionlessdata/tabulator-py/blob/master/tabulator/parsers/api.py)
     - [Processor API](https://github.com/frictionlessdata/tabulator-py/blob/master/tabulator/processors/api.py)
 
+## Changelog
+
+- 0.5.0
+  - BREAKING CHANGE: added `loaders.API.source` abstract property
+
 ## Contributing
 
 Please read the contribution guideline:
diff --git a/examples/__init__.py b/examples/__init__.py
new file mode 100644
index 0000000..e69de29
diff --git a/examples/topen.py b/examples/topen.py
index 2fd96b8..71cb06f 100644
--- a/examples/topen.py
+++ b/examples/topen.py
@@ -6,8 +6,6 @@ from __future__ import unicode_literals
 
 import io
 import sys
-
-sys.path.insert(0, '.')
 from tabulator import topen, loaders, parsers, processors
 
 
@@ -84,6 +82,22 @@ with topen(source, with_headers=True) as table:
         print(row)
 
 
+print('\nUsage of native lists:')
+source = [['id', 'name'], ['1', 'english'], ('2', '中国人')]
+with topen(source, with_headers=True) as table:
+    print(table.headers)
+    for row in table:
+        print(row)
+
+
+print('\nUsage of native lists (keyed):')
+source = [{'id': '1', 'name': 'english'}, {'id': '2', 'name': '中国人'}]
+with topen(source, with_headers=True) as table:
+    print(table.headers)
+    for row in table:
+        print(row)
+
+
 print('\nTable reset and read limit:')
 source = 'data/table.csv'
 with topen(source, with_headers=True) as table:
diff --git a/tabulator/VERSION b/tabulator/VERSION
index 1d0ba9e..8f0916f 100644
--- a/tabulator/VERSION
+++ b/tabulator/VERSION
@@ -1,1 +1,1 @@
-0.4.0
+0.5.0
diff --git a/tabulator/helpers.py b/tabulator/helpers.py
index ae1c807..0cb876e 100644
--- a/tabulator/helpers.py
+++ b/tabulator/helpers.py
@@ -4,15 +4,14 @@ from __future__ import print_function
 from __future__ import absolute_import
 from __future__ import unicode_literals
 
+import os
 import re
 import ast
+import six
 from chardet.universaldetector import UniversalDetector
 from six.moves.urllib.parse import urlparse
-
 from . import errors
 
-CHARSET_DETECTION_MAX_LINES = 1000
-CHARSET_DETECTION_MIN_CONFIDENCE = 0.5
 
 # Module API
 
@@ -25,11 +24,13 @@ def detect_scheme(source):
     """
     if hasattr(source, 'read'):
         scheme = 'stream'
-    else:
+    elif isinstance(source, six.string_types):
         match = re.search(r'^([a-zA-Z]{2,}):\/{2}', source)
         if not match:
             return None
         scheme = match.group(1).lower()
+    else:
+        scheme = 'native'
     return scheme
 
 
@@ -41,20 +42,23 @@ def detect_format(source):
     """
     if hasattr(source, 'read'):
         format = ''
-    else:
+    elif isinstance(source, six.string_types):
         parsed_source = urlparse(source)
-        path = parsed_source.path
-        if not path:
-            # FIXME: This supports valid paths like file://foo.csv, but also
-            # invalid ones like http://foo.csv
-            path = parsed_source.netloc
-        format = path.split('.')[-1].lower()
+        path = parsed_source.path or parsed_source.netloc
+        format = os.path.splitext(path)[1]
+        if not format:
+            return None
+        format = format[1:].lower()
+    else:
+        format = 'native'
     return format
 
 
 def detect_encoding(bytes):
     """Detect encoding of a byte stream.
     """
+    CHARSET_DETECTION_MAX_LINES = 1000
+    CHARSET_DETECTION_MIN_CONFIDENCE = 0.5
     detector = UniversalDetector()
     num_lines = CHARSET_DETECTION_MAX_LINES
     while num_lines > 0:
diff --git a/tabulator/loaders/__init__.py b/tabulator/loaders/__init__.py
index 1e77802..de69435 100644
--- a/tabulator/loaders/__init__.py
+++ b/tabulator/loaders/__init__.py
@@ -6,6 +6,7 @@ from __future__ import unicode_literals
 
 from .api import Loader as API
 from .file import FileLoader as File
+from .native import NativeLoader as Native
 from .stream import StreamLoader as Stream
 from .text import TextLoader as Text
 from .web import WebLoader as Web
diff --git a/tabulator/loaders/api.py b/tabulator/loaders/api.py
index edd6195..8d13511 100644
--- a/tabulator/loaders/api.py
+++ b/tabulator/loaders/api.py
@@ -14,14 +14,10 @@ from abc import ABCMeta, abstractmethod
 class Loader(object):
     """Loader representation.
 
-    Parameters
-    ----------
-    source: str
-        Table source.
-    encoding: str
-        Table encoding.
-    options: dict
-        Loader options.
+    Args:
+        source(str): table source
+        encoding(str): table encoding
+        options(dict): loader options
 
     """
 
@@ -35,22 +31,30 @@ class Loader(object):
     def load(self, mode):
         """Return byte/text stream file-like object.
 
-        Parameters
-        ----------
-        mode: str
-            Text stream mode: 't' or 'b'.
+        Args:
+            mode(str): text stream mode: 't' or 'b'
 
-        Returns
-        -------
-        file-like
-            File-like object of byte/text stream.
+        Returns:
+            file-like: file-like object of byte/text stream
 
         """
         pass  # pragma: no cover
 
     @property
+    @abstractmethod
+    def source(self):
+        """mixed: passed by user source
+
+        This property returns source set by user
+        on Loader creation.
+
+        """
+        pass  # pragma: no cover
+
+    @property
+    @abstractmethod
     def encoding(self):
-        """User defined encoding.
+        """str: user defined encoding
 
         This property returns encoding set by user
         on Loader creation. No detection here.
diff --git a/tabulator/loaders/file.py b/tabulator/loaders/file.py
index 78e4f05..63f3359 100644
--- a/tabulator/loaders/file.py
+++ b/tabulator/loaders/file.py
@@ -49,6 +49,10 @@ class FileLoader(api.Loader):
             message = 'Mode %s is not supported' % mode
             raise errors.Error(message)
 
+    @property
+    def source(self):
+        return self.__source
+
     @property
     def encoding(self):
         return self.__encoding
diff --git a/tabulator/loaders/native.py b/tabulator/loaders/native.py
new file mode 100644
index 0000000..dbb4308
--- /dev/null
+++ b/tabulator/loaders/native.py
@@ -0,0 +1,35 @@
+# -*- coding: utf-8 -*-
+from __future__ import division
+from __future__ import print_function
+from __future__ import absolute_import
+from __future__ import unicode_literals
+
+import io
+
+from .. import errors, helpers
+from . import api
+
+
+# Module API
+
+class NativeLoader(api.Loader):
+    """Null loader to pass python native lists.
+    """
+
+    # Public
+
+    def __init__(self, source, encoding=None, **options):
+        self.__source = source
+        self.__encoding = encoding
+        self.__options = options
+
+    def load(self, mode):
+        raise errors.Error('NativeLoader doesn\'t support load method')
+
+    @property
+    def source(self):
+        return self.__source
+
+    @property
+    def encoding(self):
+        return self.__encoding
diff --git a/tabulator/loaders/stream.py b/tabulator/loaders/stream.py
index 971f82a..a3ceeb6 100644
--- a/tabulator/loaders/stream.py
+++ b/tabulator/loaders/stream.py
@@ -50,6 +50,10 @@ class StreamLoader(api.Loader):
             message = 'Mode %s is not supported' % mode
             raise errors.Error(message)
 
+    @property
+    def source(self):
+        return self.__source
+
     @property
     def encoding(self):
         return self.__encoding
diff --git a/tabulator/loaders/text.py b/tabulator/loaders/text.py
index 13f5ab3..f9d0064 100644
--- a/tabulator/loaders/text.py
+++ b/tabulator/loaders/text.py
@@ -53,6 +53,10 @@ class TextLoader(api.Loader):
             message = 'Mode %s is not supported' % mode
             raise errors.Error(message)
 
+    @property
+    def source(self):
+        return self.__source
+
     @property
     def encoding(self):
         return self.__encoding
diff --git a/tabulator/loaders/web.py b/tabulator/loaders/web.py
index 3e925c6..eddc6ed 100644
--- a/tabulator/loaders/web.py
+++ b/tabulator/loaders/web.py
@@ -61,6 +61,10 @@ class WebLoader(api.Loader):
             message = 'Mode %s is not supported' % mode
             raise errors.Error(message)
 
+    @property
+    def source(self):
+        return self.__source
+
     @property
     def encoding(self):
         return self.__encoding
diff --git a/tabulator/parsers/__init__.py b/tabulator/parsers/__init__.py
index c33e0b0..e50f288 100644
--- a/tabulator/parsers/__init__.py
+++ b/tabulator/parsers/__init__.py
@@ -6,7 +6,8 @@ from __future__ import unicode_literals
 
 from .api import Parser as API
 from .csv import CSVParser as CSV
-from .tsv import TSVParser as TSV
 from .excel import ExcelParser as Excel
 from .excelx import ExcelxParser as Excelx
 from .json import JSONParser as JSON
+from .native import NativeParser as Native
+from .tsv import TSVParser as TSV
diff --git a/tabulator/parsers/json.py b/tabulator/parsers/json.py
index 316a36f..5c8787c 100644
--- a/tabulator/parsers/json.py
+++ b/tabulator/parsers/json.py
@@ -53,7 +53,7 @@ class JSONParser(api.Parser):
             prefix = '%s.item' % self.__path
         items = ijson.items(self.__chars, prefix)
         for item in items:
-            if isinstance(item, list):
+            if isinstance(item, (tuple, list)):
                 yield (None, tuple(item))
             elif isinstance(item, dict):
                 keys = []
diff --git a/tabulator/parsers/native.py b/tabulator/parsers/native.py
new file mode 100644
index 0000000..8fbffd8
--- /dev/null
+++ b/tabulator/parsers/native.py
@@ -0,0 +1,62 @@
+# -*- coding: utf-8 -*-
+from __future__ import division
+from __future__ import print_function
+from __future__ import absolute_import
+from __future__ import unicode_literals
+
+import csv
+import six
+from codecs import iterencode
+from .. import errors
+from .. import helpers
+from . import api
+
+
+# Module API
+
+class NativeParser(api.Parser):
+    """Parser to provide support for python native lists.
+    """
+
+    # Public
+
+    def __init__(self, **options):
+        self.__options = options
+        self.__loader = None
+        self.__items = None
+
+    def open(self, loader):
+        self.close()
+        self.__loader = loader
+        self.reset()
+
+    def close(self):
+        pass
+
+    @property
+    def closed(self):
+        return False
+
+    @property
+    def items(self):
+        return self.__items
+
+    def reset(self):
+        self.__items = self.__emit_items()
+
+    # Private
+
+    def __emit_items(self):
+        items = self.__loader.source
+        for item in items:
+            if isinstance(item, (tuple, list)):
+                yield (None, tuple(item))
+            elif isinstance(item, dict):
+                keys = []
+                values = []
+                for key in sorted(item.keys()):
+                    keys.append(key)
+                    values.append(item[key])
+                yield (tuple(keys), tuple(values))
+            else:
+                raise errors.Error('Native item has to be tuple, list or dict')
diff --git a/tabulator/topen.py b/tabulator/topen.py
index db63cfa..3e37834 100644
--- a/tabulator/topen.py
+++ b/tabulator/topen.py
@@ -37,11 +37,13 @@ def topen(source,
     scheme: str
         Scheme of source:
             - file (default)
+            - stream
             - text
             - http
             - https
             - ftp
             - ftps
+            - native
     format: str
         Format of source:
             - None (detect)
@@ -50,6 +52,7 @@ def topen(source,
             - json
             - xls
             - xlsx
+            - native
     encoding: str
         Encoding of source:
             - None (detect)
@@ -135,6 +138,7 @@ _LOADERS = {
     'ftps': loaders.Web,
     'http': loaders.Web,
     'https': loaders.Web,
+    'native': loaders.Native,
 }
 
 _PARSERS = {
@@ -143,4 +147,5 @@ _PARSERS = {
     'xls': parsers.Excel,
     'xlsx': parsers.Excelx,
     'json': parsers.JSON,
+    'native': parsers.Native,
 }


Test patch:
diff --git a/tests/test_topen.py b/tests/test_topen.py
index 0b4700b..07f354b 100644
--- a/tests/test_topen.py
+++ b/tests/test_topen.py
@@ -140,6 +140,50 @@ class Test_topen(unittest.TestCase):
         assert table.headers is None
         assert table.read() == [('id', 'name'), (1.0, 'english'), (2.0, '中国人')]
 
+    def test_native(self):
+
+        # Get table
+        source = [['id', 'name'], ['1', 'english'], ('2', '中国人')]
+        table = topen(source)
+
+        # Make assertions
+        assert table.headers is None
+        assert table.read() == [('id', 'name'), ('1', 'english'), ('2', '中国人')]
+
+    def test_native_iterator(self):
+
+        # Get table
+        source = iter([['id', 'name'], ['1', 'english'], ('2', '中国人')])
+        table = topen(source)
+
+        # Make assertions
+        assert table.headers is None
+        assert table.read() == [('id', 'name'), ('1', 'english'), ('2', '中国人')]
+
+    def test_native_generator(self):
+
+        # Get table
+        def generator():
+            yield ['id', 'name']
+            yield ['1', 'english']
+            yield ('2', '中国人')
+        source = generator()
+        table = topen(source)
+
+        # Make assertions
+        assert table.headers is None
+        assert table.read() == [('id', 'name'), ('1', 'english'), ('2', '中国人')]
+
+    def test_native_keyed(self):
+
+        # Get table
+        source = [{'id': '1', 'name': 'english'}, {'id': '2', 'name': '中国人'}]
+        table = topen(source, scheme='native', format='native')
+
+        # Make assertions
+        assert table.headers is None
+        assert table.read() == [('1', 'english'), ('2', '中国人')]
+
     # Tests [processors]
 
     def test_headers(self):
@@ -192,6 +236,16 @@ class Test_topen(unittest.TestCase):
         assert contents[1].get('country') == 'Brazil'
         assert contents[1].get('value') == 'Brazil'
 
+    def test_headers_native(self):
+
+        # Get table
+        source = [{'id': '1', 'name': 'english'}, {'id': '2', 'name': '中国人'}]
+        table = topen(source, with_headers=True)
+
+        # Make assertions
+        assert table.headers == ('id', 'name')
+        assert table.read() == [('1', 'english'), ('2', '中国人')]
+
     # Tests [reset]
 
     def test_reset(self):