Repository: frictionlessdata/tabulator-py (python)

Issue:
importing tabulator takes long time
When tabulator gets imported, there is a delay, which is very noticable in the command-line tool, that  uses tabulator. On the laptop it is almost 3 seconds, on the desktop about 1.5 seconds. The cli imports tabulator and jsontablechema through datapackage-py. Commenting out both of them from datapackage-py helps to make cli launch instantly.
It would be nice if anything can be done about it. Like doing most heavy stuff lazily, not at import time.

Pull request:
Improved import time
# Overview

- loaders/parsers/writers are now lazy imported
- dependencies imports have been moved to actual helper functions
- minor refactoring: added config module and default values to signatures, writers.API update 

# Issues

- fixes #106
- fixes https://github.com/frictionlessdata/jsontableschema-py/issues/118

Patch:
diff --git a/tabulator/config.py b/tabulator/config.py
new file mode 100644
index 0000000..cf5b99e
--- /dev/null
+++ b/tabulator/config.py
@@ -0,0 +1,36 @@
+# -*- coding: utf-8 -*-
+from __future__ import division
+from __future__ import print_function
+from __future__ import absolute_import
+from __future__ import unicode_literals
+
+
+# Module API
+
+DEFAULT_SCHEME = 'file'
+DEFAULT_ENCODING = 'utf-8'
+ENCODING_DETECTION_MAX_LINES = 1000
+ENCODING_DETECTION_MIN_CONFIDENCE = 0.5
+LOADERS = {
+    'file': 'tabulator.loaders.file.FileLoader',
+    'ftp': 'tabulator.loaders.web.WebLoader',
+    'ftps': 'tabulator.loaders.web.WebLoader',
+    'http': 'tabulator.loaders.web.WebLoader',
+    'https': 'tabulator.loaders.web.WebLoader',
+    'native': 'tabulator.loaders.native.NativeLoader',
+    'stream': 'tabulator.loaders.stream.StreamLoader',
+    'text': 'tabulator.loaders.text.TextLoader',
+}
+
+PARSERS = {
+    'csv': 'tabulator.parsers.csv.CSVParser',
+    'json': 'tabulator.parsers.json.JSONParser',
+    'native': 'tabulator.parsers.native.NativeParser',
+    'tsv': 'tabulator.parsers.tsv.TSVParser',
+    'xls': 'tabulator.parsers.excel.ExcelParser',
+    'xlsx': 'tabulator.parsers.excelx.ExcelxParser',
+}
+
+WRITERS = {
+    'csv': 'tabulator.writers.csv.CSVWriter',
+}
diff --git a/tabulator/helpers.py b/tabulator/helpers.py
index 454e741..52619d6 100644
--- a/tabulator/helpers.py
+++ b/tabulator/helpers.py
@@ -4,24 +4,18 @@ from __future__ import print_function
 from __future__ import absolute_import
 from __future__ import unicode_literals
 
-import codecs
 import os
 import re
 import six
-import requests.utils
-from bs4 import BeautifulSoup
+import codecs
+from importlib import import_module
 from six.moves.urllib.parse import urlparse, urlunparse
-from chardet.universaldetector import UniversalDetector
 from . import exceptions
+from . import config
 
 
 # Module API
 
-DEFAULT_SCHEME = 'file'
-DEFAULT_ENCODING = 'utf-8'
-DEFAULT_SAMPLE_SIZE = 100
-
-
 def detect_scheme(source):
     """Detect scheme by source.
 
@@ -64,6 +58,8 @@ def detect_format(source):
 def detect_encoding(bytes, encoding=None):
     """Detect encoding of a byte stream.
     """
+    # To reduce tabulator import time
+    from chardet.universaldetector import UniversalDetector
     if encoding is not None:
         if encoding.lower() == 'utf-8':
             prefix = bytes.read(len(codecs.BOM_UTF8))
@@ -71,11 +67,8 @@ def detect_encoding(bytes, encoding=None):
                 encoding = 'utf-8-sig'
             bytes.seek(0)
         return encoding
-
-    CHARDET_DETECTION_MAX_LINES = 1000
-    CHARDET_DETECTION_MIN_CONFIDENCE = 0.5
     detector = UniversalDetector()
-    num_lines = CHARDET_DETECTION_MAX_LINES
+    num_lines = config.ENCODING_DETECTION_MAX_LINES
     while num_lines > 0:
         line = bytes.readline()
         detector.feed(line)
@@ -87,17 +80,19 @@ def detect_encoding(bytes, encoding=None):
     confidence = detector.result['confidence']
     encoding = detector.result['encoding']
     # Do not use if not confident
-    if confidence < CHARDET_DETECTION_MIN_CONFIDENCE:
-        encoding = DEFAULT_ENCODING
+    if confidence < config.ENCODING_DETECTION_MIN_CONFIDENCE:
+        encoding = config.DEFAULT_ENCODING
     # Default to utf-8 for safety
     if encoding == 'ascii':
-        encoding = DEFAULT_ENCODING
+        encoding = config.DEFAULT_ENCODING
     return encoding
 
 
 def detect_html(text):
     """Detect if text is HTML.
     """
+    # To reduce tabulator import time
+    from bs4 import BeautifulSoup
     return bool(BeautifulSoup(text, 'html.parser').find())
 
 
@@ -137,6 +132,8 @@ def requote_uri(uri):
         uri (str): uri to requote
 
     """
+    # To reduce tabulator import time
+    import requests.utils
     if six.PY2:
         def url_encode_non_ascii(bytes):
             pattern = '[\x80-\xFF]'
@@ -148,3 +145,16 @@ def requote_uri(uri):
             else url_encode_non_ascii(part.encode('utf-8'))
             for index, part in enumerate(parts))
     return requests.utils.requote_uri(uri)
+
+
+def import_attribute(path):
+    """Import attribute by path.
+
+    Args:
+        path (str): in a form `package.module.attribute`
+
+    """
+    module_name, attribute_name = path.rsplit('.', 1)
+    module = import_module(module_name)
+    attribute = getattr(module, attribute_name)
+    return attribute
diff --git a/tabulator/loaders/__init__.py b/tabulator/loaders/__init__.py
index de69435..e69de29 100644
--- a/tabulator/loaders/__init__.py
+++ b/tabulator/loaders/__init__.py
@@ -1,12 +0,0 @@
-# -*- coding: utf-8 -*-
-from __future__ import division
-from __future__ import print_function
-from __future__ import absolute_import
-from __future__ import unicode_literals
-
-from .api import Loader as API
-from .file import FileLoader as File
-from .native import NativeLoader as Native
-from .stream import StreamLoader as Stream
-from .text import TextLoader as Text
-from .web import WebLoader as Web
diff --git a/tabulator/loaders/text.py b/tabulator/loaders/text.py
index ac98605..bc457ee 100644
--- a/tabulator/loaders/text.py
+++ b/tabulator/loaders/text.py
@@ -6,7 +6,7 @@ from __future__ import unicode_literals
 
 import io
 from .. import exceptions
-from .. import helpers
+from .. import config
 from . import api
 
 
@@ -30,7 +30,7 @@ class TextLoader(api.Loader):
 
         # Prepare encoding
         if encoding is None:
-            encoding = helpers.DEFAULT_ENCODING
+            encoding = config.DEFAULT_ENCODING
 
         # Prepare bytes
         bytes = io.BufferedRandom(io.BytesIO())
diff --git a/tabulator/parsers/__init__.py b/tabulator/parsers/__init__.py
index e50f288..e69de29 100644
--- a/tabulator/parsers/__init__.py
+++ b/tabulator/parsers/__init__.py
@@ -1,13 +0,0 @@
-# -*- coding: utf-8 -*-
-from __future__ import division
-from __future__ import print_function
-from __future__ import absolute_import
-from __future__ import unicode_literals
-
-from .api import Parser as API
-from .csv import CSVParser as CSV
-from .excel import ExcelParser as Excel
-from .excelx import ExcelxParser as Excelx
-from .json import JSONParser as JSON
-from .native import NativeParser as Native
-from .tsv import TSVParser as TSV
diff --git a/tabulator/stream.py b/tabulator/stream.py
index c99412c..26b3811 100644
--- a/tabulator/stream.py
+++ b/tabulator/stream.py
@@ -5,12 +5,11 @@ from __future__ import absolute_import
 from __future__ import unicode_literals
 
 import six
+from copy import copy
 from itertools import chain
-from . import loaders
-from . import parsers
-from . import writers
-from . import helpers
 from . import exceptions
+from . import helpers
+from . import config
 
 
 # Module API
@@ -75,20 +74,10 @@ class Stream(object):
                  scheme=None,
                  format=None,
                  encoding=None,
-                 post_parse=None,
-                 sample_size=None,
-                 loader_options=None,
-                 parser_options=None,):
-
-        # Defaults
-        if loader_options is None:
-            loader_options = {}
-        if parser_options is None:
-            parser_options = {}
-        if post_parse is None:
-            post_parse = []
-        if sample_size is None:
-            sample_size = helpers.DEFAULT_SAMPLE_SIZE
+                 post_parse=[],
+                 sample_size=100,
+                 loader_options={},
+                 parser_options={}):
 
         # Headers
         self.__headers = None
@@ -104,24 +93,26 @@ class Stream(object):
 
         # Loader
         if scheme is None:
-            scheme = helpers.detect_scheme(source) or helpers.DEFAULT_SCHEME
-        if scheme not in _LOADERS:
+            scheme = helpers.detect_scheme(source) or config.DEFAULT_SCHEME
+        if scheme not in config.LOADERS:
             message = 'Scheme "%s" is not supported' % scheme
             raise exceptions.LoadingError(message)
-        self.__loader = _LOADERS[scheme](**loader_options)
+        loader_class = helpers.import_attribute(config.LOADERS[scheme])
+        self.__loader = loader_class(**loader_options)
 
         # Parser
         if format is None:
             format = helpers.detect_format(source)
-        if format not in _PARSERS:
+        if format not in config.PARSERS:
             message = 'Format "%s" is not supported' % format
             raise exceptions.ParsingError(message)
-        self.__parser = _PARSERS[format](**parser_options)
+        parser_class = helpers.import_attribute(config.PARSERS[format])
+        self.__parser = parser_class(**parser_options)
 
         # Attributes
         self.__source = source
         self.__encoding = encoding
-        self.__post_parse = post_parse
+        self.__post_parse = copy(post_parse)
         self.__sample_size = sample_size
         self.__sample_extended_rows = []
         self.__number = 0
@@ -234,19 +225,20 @@ class Stream(object):
                 break
         return result
 
-    def save(self, target, format=None,  encoding=None, **writer_options):
+    def save(self, target, format=None,  encoding=None, **options):
         """Save stream to filesystem.
         """
         if encoding is None:
-            encoding = helpers.DEFAULT_ENCODING
+            encoding = config.DEFAULT_ENCODING
         if format is None:
             format = helpers.detect_format(target)
-        if format not in _WRITERS:
+        if format not in config.WRITERS:
             message = 'Format "%s" is not supported' % format
             raise exceptions.WritingError(message)
         extended_rows = self.iter(extended=True)
-        writer = _WRITERS[format]()
-        writer.write(target, encoding, extended_rows, **writer_options)
+        writer_class = helpers.import_attribute(config.WRITERS[format])
+        writer = writer_class(**options)
+        writer.write(target, encoding, extended_rows)
 
     # Private
 
@@ -301,30 +293,3 @@ class Stream(object):
         for processor in processors:
             iterator = processor(iterator)
         return iterator
-
-
-# Internal
-
-_LOADERS = {
-    'file': loaders.File,
-    'stream': loaders.Stream,
-    'text': loaders.Text,
-    'ftp': loaders.Web,
-    'ftps': loaders.Web,
-    'http': loaders.Web,
-    'https': loaders.Web,
-    'native': loaders.Native,
-}
-
-_PARSERS = {
-    'csv': parsers.CSV,
-    'tsv': parsers.TSV,
-    'xls': parsers.Excel,
-    'xlsx': parsers.Excelx,
-    'json': parsers.JSON,
-    'native': parsers.Native,
-}
-
-_WRITERS = {
-    'csv': writers.CSV,
-}
diff --git a/tabulator/topen.py b/tabulator/topen.py
index fb61548..db232dc 100644
--- a/tabulator/topen.py
+++ b/tabulator/topen.py
@@ -16,10 +16,10 @@ def topen(source,
           scheme=None,
           format=None,
           encoding=None,
-          post_parse=None,
-          sample_size=None,
-          loader_options=None,
-          parser_options=None,
+          post_parse=[],
+          sample_size=100,
+          loader_options={},
+          parser_options={},
           # DEPRECATED [v0.5-v1)
           loader_class=None,
           parser_class=None,
diff --git a/tabulator/writers/__init__.py b/tabulator/writers/__init__.py
index 0d5ece7..e69de29 100644
--- a/tabulator/writers/__init__.py
+++ b/tabulator/writers/__init__.py
@@ -1,8 +0,0 @@
-# -*- coding: utf-8 -*-
-from __future__ import division
-from __future__ import print_function
-from __future__ import absolute_import
-from __future__ import unicode_literals
-
-from .api import Writer as API
-from .csv import CSVWriter as CSV
diff --git a/tabulator/writers/api.py b/tabulator/writers/api.py
index 94dd317..0ec43a7 100644
--- a/tabulator/writers/api.py
+++ b/tabulator/writers/api.py
@@ -13,12 +13,16 @@ from abc import ABCMeta, abstractmethod
 @add_metaclass(ABCMeta)
 class Writer(object):
     """Writer representation.
+
+    Args:
+        options(dict): writer options
+
     """
 
     # Public
 
     @abstractmethod
-    def write(self, target, encoding, extended_rows, **options):
+    def write(self, target, encoding, extended_rows):
         """Write tabular data to target.
         """
         pass
diff --git a/tabulator/writers/csv.py b/tabulator/writers/csv.py
index ada4558..cf416ce 100644
--- a/tabulator/writers/csv.py
+++ b/tabulator/writers/csv.py
@@ -18,10 +18,14 @@ class CSVWriter(api.Writer):
 
     # Public
 
-    def write(self, target, encoding, extended_rows, **options):
+    def __init__(self, **options):
+        self.__options = options
+
+    def write(self, target, encoding, extended_rows):
         helpers.ensure_dir(target)
         with io.open(target, 'wb') as file:
-            writer = unicodecsv.writer(file, encoding=encoding, **options)
+            writer = unicodecsv.writer(
+                file, encoding=encoding, **self.__options)
             iterator = enumerate(extended_rows, start=1)
             for count, (_, headers, row) in iterator:
                 if count == 1 and headers:


Test patch:
diff --git a/tests/loaders/test_file.py b/tests/loaders/test_file.py
index f6482d7..fc4ba15 100644
--- a/tests/loaders/test_file.py
+++ b/tests/loaders/test_file.py
@@ -4,19 +4,19 @@ from __future__ import print_function
 from __future__ import absolute_import
 from __future__ import unicode_literals
 
-from tabulator import loaders
+from tabulator.loaders.file import FileLoader
 
 
 # Tests
 
 def test_load_t():
-    loader = loaders.File()
+    loader = FileLoader()
     chars = loader.load('data/table.csv', 'utf-8', mode='t')
     assert chars.read() == 'id,name\n1,english\n2,中国人\n'
 
 
 def test_load_b():
     spec = '中国人'.encode('utf-8')
-    loader = loaders.File()
+    loader = FileLoader()
     chars = loader.load('data/table.csv', 'utf-8', mode='b')
     assert chars.read() == b'id,name\n1,english\n2,' + spec + b'\n'
diff --git a/tests/loaders/test_text.py b/tests/loaders/test_text.py
index 79a4b35..fe892a2 100644
--- a/tests/loaders/test_text.py
+++ b/tests/loaders/test_text.py
@@ -4,7 +4,7 @@ from __future__ import print_function
 from __future__ import absolute_import
 from __future__ import unicode_literals
 
-from tabulator import loaders
+from tabulator.loaders.text import TextLoader
 
 
 # Constants
@@ -15,12 +15,12 @@ SOURCE = 'id,name\n1,english\n2,中国人\n'
 # Tests
 
 def test_load_t():
-    loader = loaders.Text()
+    loader = TextLoader()
     chars = loader.load(SOURCE, 'utf-8', mode='t')
     assert chars.read() == 'id,name\n1,english\n2,中国人\n'
 
 def test_load_b():
     spec = '中国人'.encode('utf-8')
-    loader = loaders.Text()
+    loader = TextLoader()
     chars = loader.load(SOURCE, 'utf-8', mode='b')
     assert chars.read() == b'id,name\n1,english\n2,' + spec + b'\n'
diff --git a/tests/loaders/test_web.py b/tests/loaders/test_web.py
index 2a32fe1..1dfdc88 100644
--- a/tests/loaders/test_web.py
+++ b/tests/loaders/test_web.py
@@ -4,7 +4,7 @@ from __future__ import print_function
 from __future__ import absolute_import
 from __future__ import unicode_literals
 
-from tabulator import loaders
+from tabulator.loaders.web import WebLoader
 
 
 # Constants
@@ -15,13 +15,13 @@ SOURCE = 'https://raw.githubusercontent.com/frictionlessdata/tabulator-py/master
 # Tests
 
 def test_load_t():
-    loader = loaders.Web()
+    loader = WebLoader()
     chars = loader.load(SOURCE, 'utf-8', mode='t')
     assert chars.read() == 'id,name\n1,english\n2,中国人\n'
 
 
 def test_load_b():
     spec = '中国人'.encode('utf-8')
-    loader = loaders.Web()
+    loader = WebLoader()
     chars = loader.load(SOURCE, 'utf-8', mode='b')
     assert chars.read() == b'id,name\n1,english\n2,' + spec + b'\n'
diff --git a/tests/parsers/test_csv.py b/tests/parsers/test_csv.py
index 952ca15..7461a19 100644
--- a/tests/parsers/test_csv.py
+++ b/tests/parsers/test_csv.py
@@ -6,7 +6,7 @@ from __future__ import unicode_literals
 
 import io
 from mock import Mock
-from tabulator import parsers
+from tabulator.parsers.csv import CSVParser
 
 
 # Tests
@@ -17,7 +17,7 @@ def test_csv_parser():
     encoding = None
     loader = Mock()
     loader.load = Mock(return_value=io.open(source, encoding='utf-8'))
-    parser = parsers.CSV()
+    parser = CSVParser()
 
     assert parser.closed
     parser.open(source, encoding, loader)
diff --git a/tests/parsers/test_excel.py b/tests/parsers/test_excel.py
index 630fd94..468377e 100644
--- a/tests/parsers/test_excel.py
+++ b/tests/parsers/test_excel.py
@@ -7,6 +7,7 @@ from __future__ import unicode_literals
 import io
 from mock import Mock
 from tabulator import parsers
+from tabulator.parsers.excel import ExcelParser
 
 
 # Tests
@@ -17,7 +18,7 @@ def test_excel_parser():
     encoding = None
     loader = Mock()
     loader.load = Mock(return_value=io.open(source, 'rb'))
-    parser = parsers.Excel()
+    parser = ExcelParser()
 
     assert parser.closed
     parser.open(source, encoding, loader)
diff --git a/tests/parsers/test_excelx.py b/tests/parsers/test_excelx.py
index 1124644..e5e21b4 100644
--- a/tests/parsers/test_excelx.py
+++ b/tests/parsers/test_excelx.py
@@ -6,7 +6,7 @@ from __future__ import unicode_literals
 
 import io
 from mock import Mock
-from tabulator import parsers
+from tabulator.parsers.excelx import ExcelxParser
 
 
 # Tests
@@ -17,7 +17,7 @@ def test_excelx_parser():
     encoding = None
     loader = Mock()
     loader.load = Mock(return_value=io.open(source, 'rb'))
-    parser = parsers.Excelx()
+    parser = ExcelxParser()
 
     assert parser.closed
     parser.open(source, encoding, loader)
diff --git a/tests/parsers/test_json.py b/tests/parsers/test_json.py
index 542e225..0f32288 100644
--- a/tests/parsers/test_json.py
+++ b/tests/parsers/test_json.py
@@ -6,7 +6,7 @@ from __future__ import unicode_literals
 
 import io
 from mock import Mock
-from tabulator import parsers
+from tabulator.parsers.json import JSONParser
 
 
 # Tests
@@ -17,7 +17,7 @@ def test_json_parser():
     encoding = None
     loader = Mock()
     loader.load = Mock(return_value=io.open(source, 'rb'))
-    parser = parsers.JSON()
+    parser = JSONParser()
 
     assert parser.closed
     parser.open(source, encoding, loader)
diff --git a/tests/parsers/test_tsv.py b/tests/parsers/test_tsv.py
index 597d220..f0c3d35 100644
--- a/tests/parsers/test_tsv.py
+++ b/tests/parsers/test_tsv.py
@@ -6,7 +6,7 @@ from __future__ import unicode_literals
 
 import io
 from mock import Mock
-from tabulator import parsers
+from tabulator.parsers.tsv import TSVParser
 
 
 # Tests
@@ -17,7 +17,7 @@ def test_tsv_parser():
     encoding = None
     loader = Mock()
     loader.load = Mock(return_value=io.open(source))
-    parser = parsers.TSV()
+    parser = TSVParser()
 
     assert parser.closed
     parser.open(source, encoding, loader)
diff --git a/tests/test_helpers.py b/tests/test_helpers.py
index 05dc5f8..e54ffac 100644
--- a/tests/test_helpers.py
+++ b/tests/test_helpers.py
@@ -79,3 +79,12 @@ def test_requote_uri():
     url2 = 'http://data.defra.gov.uk/ops/government_procurement_card/over_%C2%A3500_GPC_apr_2013.csv'
     assert helpers.requote_uri(url) == url
     assert helpers.requote_uri(url1) == url2
+
+
+def text_import_attribute():
+    assert helpers.import_attribute('tabulator.helpers') == helpers
+
+
+def text_import_attribute_import_error():
+    with pytest.raises(ImportError):
+        helpers.import_attribute('tabulator.bad_name')
diff --git a/tests/test_topen.py b/tests/test_topen.py
index d90a617..4c39009 100644
--- a/tests/test_topen.py
+++ b/tests/test_topen.py
@@ -9,7 +9,8 @@ import ast
 import sys
 import six
 import pytest
-from tabulator import topen, parsers, exceptions
+from tabulator import topen, exceptions
+from tabulator.parsers.csv import CSVParser
 
 
 # Constants
@@ -33,7 +34,7 @@ def test_file_csv_parser_options():
 
     # Get table
     table = topen('data/table.csv',
-            parser_options={'constructor': parsers.CSV})
+            parser_options={'constructor': CSVParser})
 
     # Make assertions
     assert table.headers is None
@@ -56,12 +57,11 @@ def test_file_csv_with_bom():
     assert table.read() == [['id', 'name'], ['1', 'english'], ['2', '中国人']]
 
 
-
 # DEPRECATED [v0.5-v1)
 def test_file_csv_parser_class():
 
     # Get table
-    table = topen('data/table.csv', parser_class=parsers.CSV)
+    table = topen('data/table.csv', parser_class=CSVParser)
 
     # Make assertions
     assert table.headers is None