Repository: frictionlessdata/tabulator-py
Pull request #113 (2016-10-26 09:37:31)

Add NDJSON support
NDJSON support is based on jsonlines [1] library.

[1] https://github.com/wbolster/jsonlines

It fixes https://github.com/frictionlessdata/tabulator-py/issues/32

roll: @sirex awesome! 

It think it will be good to also add a high-level test to `test_stream.py` something like:
```python
def test_stream_ndjson():
    with Stream('data/table.ndjson', headers=1) as stream:
        assert stream.headers == ['id', 'name']
        assert stream.read(keyed=True) == [
            {'id': '1', 'name': 'english'},
            {'id': '2', 'name': '中国人'}]


```
roll: Not sure what's wrong with our Travis..
roll: Thanks @sirex - you're doing a great contribution. I'll merge and release after travis finishes.

Patch:
diff --git a/README.md b/README.md
index fb1d6a9..5203d46 100644
--- a/README.md
+++ b/README.md
@@ -10,7 +10,7 @@ Consistent interface for stream reading and writing tabular data (csv/xls/json/e
 
 ## Features
 
-- supports various formats: csv/tsv/xls/xlsx/json/native/etc
+- supports various formats: csv/tsv/xls/xlsx/json/ndjson/native/etc
 - reads data from variables, filesystem or Internet
 - streams data instead of using a lot of memory
 - processes data via simple user processors
diff --git a/data/table.ndjson b/data/table.ndjson
new file mode 100644
index 0000000..0887e4d
--- /dev/null
+++ b/data/table.ndjson
@@ -0,0 +1,2 @@
+{"id":1,"name":"english"}
+{"id":2,"name":"中国人"}
diff --git a/setup.py b/setup.py
index 31bec56..4ee4ea5 100644
--- a/setup.py
+++ b/setup.py
@@ -29,6 +29,7 @@ INSTALL_REQUIRES = [
     'beautifulsoup4>=4.4,<5.0a',
     'linear-tsv>=1.0,<2.0a',
     'unicodecsv>=0.14,<1.0a',
+    'jsonlines>=1.1,<1.2',
 ]
 TESTS_REQUIRE = [
     'pylama',
diff --git a/tabulator/config.py b/tabulator/config.py
index cf5b99e..6c5acf3 100644
--- a/tabulator/config.py
+++ b/tabulator/config.py
@@ -25,6 +25,8 @@ LOADERS = {
 PARSERS = {
     'csv': 'tabulator.parsers.csv.CSVParser',
     'json': 'tabulator.parsers.json.JSONParser',
+    'jsonl': 'tabulator.parsers.ndjson.NDJSONParser',
+    'ndjson': 'tabulator.parsers.ndjson.NDJSONParser',
     'native': 'tabulator.parsers.native.NativeParser',
     'tsv': 'tabulator.parsers.tsv.TSVParser',
     'xls': 'tabulator.parsers.excel.ExcelParser',
diff --git a/tabulator/parsers/ndjson.py b/tabulator/parsers/ndjson.py
new file mode 100644
index 0000000..7080dcd
--- /dev/null
+++ b/tabulator/parsers/ndjson.py
@@ -0,0 +1,60 @@
+# -*- coding: utf-8 -*-
+from __future__ import division
+from __future__ import print_function
+from __future__ import absolute_import
+from __future__ import unicode_literals
+
+import jsonlines
+
+from .. import helpers
+from . import api
+
+
+# Module API
+
+class NDJSONParser(api.Parser):
+    """Parser to parse NDJSON data format.
+
+    See: http://specs.okfnlabs.org/ndjson/
+    """
+
+    # Public
+
+    options = [
+    ]
+
+    def __init__(self, **options):
+        self.__options = options
+        self.__extended_rows = None
+        self.__loader = None
+        self.__chars = None
+
+    @property
+    def closed(self):
+        return self.__chars is None or self.__chars.closed
+
+    def open(self, source, encoding, loader):
+        self.close()
+        self.__loader = loader
+        self.__chars = loader.load(source, encoding, mode='t')
+        self.reset()
+
+    def close(self):
+        if not self.closed:
+            self.__chars.close()
+
+    def reset(self):
+        helpers.reset_stream(self.__chars)
+        self.__extended_rows = self.__iter_extended_rows()
+
+    @property
+    def extended_rows(self):
+        return self.__extended_rows
+
+    # Private
+
+    def __iter_extended_rows(self):
+        rows = jsonlines.Reader(self.__chars)
+        for number, row in enumerate(rows, start=1):
+            keys, values = zip(*sorted(row.items()))
+            yield number, list(keys), list(values)


Test patch:
diff --git a/tests/parsers/test_ndjson.py b/tests/parsers/test_ndjson.py
new file mode 100644
index 0000000..3ae272c
--- /dev/null
+++ b/tests/parsers/test_ndjson.py
@@ -0,0 +1,45 @@
+# -*- coding: utf-8 -*-
+from __future__ import division
+from __future__ import print_function
+from __future__ import absolute_import
+from __future__ import unicode_literals
+
+import io
+from mock import Mock
+from tabulator import Stream
+from tabulator.parsers.ndjson import NDJSONParser
+
+
+# Tests
+
+def test_ndjson_parser():
+
+    source = 'data/table.ndjson'
+    encoding = None
+    loader = Mock()
+    loader.load = Mock(return_value=io.open(source, encoding='utf-8'))
+    parser = NDJSONParser()
+
+    assert parser.closed is True
+    parser.open(source, encoding, loader)
+    assert parser.closed is False
+
+    assert list(parser.extended_rows) == [
+        (1, ['id', 'name'], [1, 'english']),
+        (2, ['id', 'name'], [2, '中国人']),
+    ]
+
+    assert len(list(parser.extended_rows)) == 0
+    parser.reset()
+    assert len(list(parser.extended_rows)) == 2
+
+    parser.close()
+    assert parser.closed
+
+
+def test_stream_ndjson():
+    with Stream('data/table.ndjson', headers=1) as stream:
+        assert stream.headers == ['id', 'name']
+        assert stream.read(keyed=True) == [
+            {'id': 1, 'name': 'english'},
+            {'id': 2, 'name': '中国人'}]