Repository: mblomdahl/qsck

Issue:
Implement `deserialize` function
Hi Mats! Now when we the awesome serializer published and ready as release `qsck==0.1`, how about building upon #4 PR to make a `qsck.deserialize` function as well? Preferably with good usage docs and test coverage, if you would be so kind.. 😸 

Discussion:

Patch:
diff --git a/README.md b/README.md
index 99afb17..85f66d2 100644
--- a/README.md
+++ b/README.md
@@ -5,13 +5,18 @@ qsck
 Python library for serializing and deserializing a wonky format referred to
 as ".qs" files. For full format specification, please read through
 [tests.test_serialize](https://github.com/mblomdahl/qsck/blob/master/tests/test_serialize.py)
-test suite and extrapolate.
+and
+[tests.test_deserialize](https://github.com/mblomdahl/qsck/blob/master/tests/test_deserialize.py)
+test suites and extrapolate.
 
 
 Quick Start
 -----------
 
-**Use Python ≥ 3.6 only.** Install it: `pip3 install qsck`
+**Use Python ≥ 3.6 only.** To install it, simply:
+
+    pip3 install qsck
+
 
 ### Serializing Data
 
@@ -36,6 +41,31 @@ the command-line tool `qs-format`, one record per line:
     qs-format my-records.json > my-records.qs
 
 
+## Deserializing Data
+
+Via Python:
+
+    python3 -c "import pprint, qsck; pprint.pprint(qsck.deserialize(
+        'LOG,1553302923,first_key=some value,2nd_key={attr1=foo, \
+        attr2=bar},3rd_key={\"subKey1\":\"-3\",\"subKey2\":null},4th_key=(null)'))"
+
+
+Out comes a friendly Python collection:
+
+    ('LOG',
+     '1553302923',
+     [('first_key', 'some value'),
+      ('2nd_key', [('attr1', 'foo'), ('attr2', 'bar')]),
+      ('3rd_key', {'subKey1': '-3', 'subKey2': None}),
+      ('4th_key', None)])
+
+
+The library-provided `qs-parse` command-line tool supports deserializing a whole
+".qs" log file, emitting one JSON record per input line to stdout:
+
+    qs-parse my-records.qs > my-records.json 
+
+
 Contributing
 ------------
 
@@ -56,7 +86,12 @@ Distributing:
 Changelog
 ---------
 
-### 0.1 – Initial Release
+### `0.2` – Deserialize Added
+
+Adds support for de-serializing ".qs" records.
+
+
+### `0.1` – Initial Release
 
 Supports serializing ".qs" records.
 
diff --git a/qsck/__init__.py b/qsck/__init__.py
index 861a3e1..cec5cba 100644
--- a/qsck/__init__.py
+++ b/qsck/__init__.py
@@ -9,7 +9,8 @@ from datetime import datetime
 
 import ujson
 
-from .util import _validate_and_cast_timestamp_to_epoch_str
+from .util import (_validate_and_cast_timestamp_to_epoch_str,
+                   _reconstruct_key_value_pairs)
 
 
 def serialize(identifier: str, timestamp, key_value_pairs: []) -> str:
@@ -46,4 +47,12 @@ def deserialize(qs_row: str) -> (str, datetime, []):
     """Parse `qs_row`, return as a identifier-timestamp-key_value_pairs 3-tuple.
     """
 
-    raise NotImplemented
+    input_components = qs_row.rstrip().split(',')
+    if len(input_components) < 3:
+        raise AssertionError(f'Malformatted input row {qs_row!r}')
+
+    identifier, timestamp = input_components[:2]
+
+    key_value_thingies = _reconstruct_key_value_pairs(input_components[2:])
+
+    return identifier, timestamp, key_value_thingies
diff --git a/qsck/format_cli.py b/qsck/format_cli.py
index e05b074..523dc0c 100644
--- a/qsck/format_cli.py
+++ b/qsck/format_cli.py
@@ -1,4 +1,4 @@
-"""Placeholder module for `qs-format` command-line tool."""
+"""Module providing the `qs-format` command-line tool."""
 
 import click
 import ujson
diff --git a/qsck/parse_cli.py b/qsck/parse_cli.py
index cc8751d..df6dd82 100644
--- a/qsck/parse_cli.py
+++ b/qsck/parse_cli.py
@@ -1,9 +1,21 @@
-"""Placeholder module for `qs-parse` command-line tool."""
+"""Module providing the `qs-parse` command-line tool."""
 
+import click
+import ujson
 
-def run():
-    print("Here is the parser!")
+from . import deserialize
+
+
+@click.command()
+@click.argument('input_qs_file', type=click.File())
+def qs_parse(input_qs_file):
+    """Reads ".qs" file, outputs one JSON record per input line to stdout."""
+
+    for row in input_qs_file.readlines():
+        input_record = deserialize(row)
+        json_output = ujson.dumps(input_record)
+        print(json_output)
 
 
 if __name__ == '__main__':
-    run()
+    qs_parse()
diff --git a/qsck/util.py b/qsck/util.py
index b76ad6f..cb744f2 100644
--- a/qsck/util.py
+++ b/qsck/util.py
@@ -1,6 +1,9 @@
 """Misc utility functions to make serialize/deserialize work."""
 
 from datetime import datetime, timezone
+from re import match
+
+import ujson
 
 
 def _validate_and_cast_timestamp_to_epoch_str(timestamp) -> str:
@@ -31,3 +34,110 @@ def _validate_and_cast_timestamp_to_epoch_str(timestamp) -> str:
 
     return str(int_timestamp)
 
+
+def _get_nested_list_pair(nested_key_value_str: str) -> (str, str):
+    nested_key_value_pair = nested_key_value_str.split('=')
+    if len(nested_key_value_pair) != 2:
+        raise AssertionError(f"Don't know what to do with "
+                             f"{nested_key_value_str!r}")
+    key, value = nested_key_value_pair
+    key: str = key.lstrip()
+
+    return key, value
+
+
+def _reconstruct_key_value_pairs(key_value_components: list) -> list:
+
+    parsed_components = []
+
+    tmp_nesting_key = None
+    tmp_nested_list_components, tmp_nested_dict_components = [], []
+    parsing_nested_list, parsing_nested_dict = False, False
+
+    for idx, thing in enumerate(key_value_components):
+        try:
+            if thing.count('=') == 1 and '={' not in thing \
+                    and not any([parsing_nested_list, parsing_nested_dict]):
+                key, value = thing.split('=')
+                if value == '(null)':
+                    value = None
+                parsed_components.append((key, value))
+
+            elif thing.count('={') == 1:  # Nested list or dict starting here.
+                tmp_nesting_key, first_nested_pair = thing.split('={')
+
+                if match(r'\s*[\w\d]+=.+', first_nested_pair):
+                    parsing_nested_list = True
+                    key, value = _get_nested_list_pair(first_nested_pair)
+
+                    if not value.endswith('}'):  # Add pair to nested list cmp.
+                        tmp_nested_list_components.append((key, value))
+
+                    else:  # Pack it up and reset local state, single-pair case.
+                        parsed_components.append((tmp_nesting_key,
+                                                  [(key, value.rstrip('}'))]))
+                        parsing_nested_list = False
+
+                elif match(r'".+":.+', first_nested_pair):
+                    parsing_nested_dict = True
+                    if not first_nested_pair.endswith('}'):
+                        # Add to nested dict components.
+                        tmp_nested_dict_components.append(first_nested_pair)
+                    else:
+                        # Pack it up and reset local state, single-pair case.
+                        parsed_components.append(
+                            (tmp_nesting_key, ujson.loads(
+                                '{%s}' % first_nested_pair.rstrip('}')))
+                        )
+                        parsing_nested_dict = False
+
+                elif first_nested_pair == '}':  # Empty nesting.
+                    parsed_components.append((tmp_nesting_key, []))
+
+                else:
+                    raise AssertionError(f"Don't know what to do with first "
+                                         f"nested pair {first_nested_pair!r}")
+
+            elif thing.endswith('}'):  # Nested list or dict ending here.
+                last_nested_pair: str = thing.rstrip('}')
+
+                if all([parsing_nested_list, len(tmp_nested_list_components),
+                        match(r'\s*[\w\d]+=.+', last_nested_pair)]):
+                    key, value = _get_nested_list_pair(last_nested_pair)
+                    parsed_components.append(
+                        (tmp_nesting_key,
+                         tmp_nested_list_components + [(key, value)])
+                    )
+                    tmp_nested_list_components = []
+                    parsing_nested_list = False
+
+                elif all([parsing_nested_dict, len(tmp_nested_dict_components),
+                          match(r'\s*"\w+":.+', last_nested_pair)]):
+                    parsed_components.append(
+                        (tmp_nesting_key, ujson.loads('{%s}' % ','.join(
+                            tmp_nested_dict_components + [last_nested_pair])))
+                    )
+                    tmp_nested_dict_components = []
+                    parsing_nested_dict = False
+
+                else:
+                    raise AssertionError(f"Don't know what to do with "
+                                         f"{last_nested_pair!r}")
+
+            elif parsing_nested_list:  # Add pair to nested list cmp.
+                intermediate_nested_pair = thing
+                key, value = _get_nested_list_pair(intermediate_nested_pair)
+                tmp_nested_list_components.append((key, value))
+
+            elif parsing_nested_dict:  # Add pair to nested list cmp.
+                intermediate_nested_pair = thing
+                tmp_nested_dict_components.append(intermediate_nested_pair)
+
+            else:
+                raise AssertionError(f"Don't know what to do with {thing!r}")
+
+        except Exception as parse_err:
+            raise parse_err.__class__(f'Error parsing {thing!r} at index {idx} '
+                                      f'({str(parse_err)})')
+
+    return parsed_components
diff --git a/setup.cfg b/setup.cfg
index 01371f7..8bb9616 100644
--- a/setup.cfg
+++ b/setup.cfg
@@ -5,5 +5,5 @@ description-file = README.md
 test = pytest
 
 [tool:pytest]
-addopts = --verbose
+addopts = -vv
 python_files = tests/*.py
diff --git a/setup.py b/setup.py
index 397218d..a797dde 100644
--- a/setup.py
+++ b/setup.py
@@ -13,7 +13,7 @@ def _read_files_from_root_dir(*file_basenames: [str]) -> str:
 
 setup(
     name='qsck',
-    version='0.1',
+    version='0.2',
     packages=find_packages(include=('qsck',)),
     url='https://github.com/mblomdahl/qsck',
     license='The Unlicense',
@@ -24,7 +24,7 @@ setup(
     long_description_content_type='text/markdown',
     entry_points={
         'console_scripts': [
-            'qs-parse = qsck.parse_cli:run',
+            'qs-parse = qsck.parse_cli:qs_parse',
             'qs-format = qsck.format_cli:qs_format'
         ]
     },


Test patch:
diff --git a/tests/test_deserialize.py b/tests/test_deserialize.py
index 2d8883b..5d83b08 100644
--- a/tests/test_deserialize.py
+++ b/tests/test_deserialize.py
@@ -1,3 +1,4 @@
+from pytest import raises
 
 from qsck import deserialize
 
@@ -5,3 +6,90 @@ from qsck import deserialize
 def test_deserialize_is_a_function():
     assert hasattr(deserialize, '__call__')
 
+
+def test_it_returns_a_3tuple_with_identifier_timestamp_and_pairs_list():
+    qs_row = 'LOG,1546902289,_model=LG-M327'
+
+    identifier, timestamp, key_value_pairs = deserialize(qs_row)
+
+    assert identifier == 'LOG'
+
+    assert timestamp == '1546902289'
+
+    assert key_value_pairs == [('_model', 'LG-M327')]
+
+
+def test_it_rejects_malformatted_qs_row():
+    qs_row = 'LOG,1546902289'
+
+    with raises(AssertionError):
+        deserialize(qs_row)
+
+
+def test_it_returns_none_values_from_funky_null_strings():
+    qs_row = 'LOG,1546902289,_app_version=(null),_model=LG-M327'
+
+    _, __, key_value_pairs = deserialize(qs_row)
+
+    assert key_value_pairs == [('_app_version', None), ('_model', 'LG-M327')]
+
+
+def test_it_parses_nested_list_content():
+    qs_row = (
+        'LOG,1546902289,_model=LG-M327,'
+        '0event_vars={subtype=disconnected},'
+        'event1_vars={},'
+        'event_vars10={batteryPc1=0.79, isCharging=false, subtype=connected, '
+        'batteryTemp=310, chargeTyp3=Not_Charging},'
+        'event11_data=BATTERY_CHANGED,'
+        'event11_time=1546901849405,'
+        'event11_vars={batteryPct=0.78, batteryTemp=308},'
+        '_rx_host=ip-10-0-1-215'
+    )
+
+    _, __, key_value_pairs_run1 = deserialize(qs_row)
+    _, __, key_value_pairs_run2 = deserialize(qs_row)
+
+    for key_value_pairs in [key_value_pairs_run1, key_value_pairs_run2]:
+        assert key_value_pairs == [
+            ('_model', 'LG-M327'),
+            ('0event_vars', [('subtype', 'disconnected')]),
+            ('event1_vars', []),
+            ('event_vars10', [('batteryPc1', '0.79'),
+                              ('isCharging', 'false'),
+                              ('subtype', 'connected'),
+                              ('batteryTemp', '310'),
+                              ('chargeTyp3', 'Not_Charging')]),
+            ('event11_data', 'BATTERY_CHANGED'),
+            ('event11_time', '1546901849405'),
+            ('event11_vars', [('batteryPct', '0.78'),
+                              ('batteryTemp', '308')]),
+            ('_rx_host', 'ip-10-0-1-215')
+        ]
+
+
+def test_it_parses_nested_dict_content():
+    qs_row = (
+        'LOG,1546902289,user=jenkins,'
+        '1nfo_healthDat4={"battery_max":0.89,"battery_max_a1":1546898400064,'
+        '"battery_min":0.78,"battery_min_at":1546901880022,'
+        '"time_charging":0,"time_discharging":60},'
+        'info_runDat4={"app_install_time":1545251927594},'
+        'time=1546902289176'
+    )
+
+    _, __, key_value_pairs_run1 = deserialize(qs_row)
+    _, __, key_value_pairs_run2 = deserialize(qs_row)
+
+    for key_value_pairs in [key_value_pairs_run1, key_value_pairs_run2]:
+        assert key_value_pairs == [
+            ('user', 'jenkins'),
+            ('1nfo_healthDat4', {"battery_max": 0.89,
+                                 "battery_max_a1": 1546898400064,
+                                 "battery_min": 0.78,
+                                 "battery_min_at": 1546901880022,
+                                 "time_charging": 0,
+                                 "time_discharging": 60}),
+            ('info_runDat4', {"app_install_time": 1545251927594}),
+            ('time', '1546902289176')
+        ]
diff --git a/tests/test_serialize.py b/tests/test_serialize.py
index 1441609..3c9fe75 100644
--- a/tests/test_serialize.py
+++ b/tests/test_serialize.py
@@ -76,6 +76,7 @@ def test_it_does_the_nested_key_value_formatting_on_root_level_list_values():
     timestamp = datetime.utcnow()
     key_value_pairs_with_list_values = [
         ('howdy', None),
+        ('empty', []),
         ('my_nest', [('sub_key1', 'foo'), ('sk2', 'bar')]),
         ('nest2', [('a', '1')]),
         ('otherStuff', 'ok')
@@ -83,7 +84,7 @@ def test_it_does_the_nested_key_value_formatting_on_root_level_list_values():
 
     qs_row = serialize(identifier, timestamp, key_value_pairs_with_list_values)
 
-    assert ',my_nest={sub_key1=foo, sk2=bar},nest2={a=1},otherStuff=' in qs_row
+    assert ',empty={},my_nest={sub_key1=foo, sk2=bar},nest2={a=1},oth' in qs_row
 
 
 def test_it_does_the_nested_key_value_formatting_on_root_level_dict_values():
@@ -91,14 +92,15 @@ def test_it_does_the_nested_key_value_formatting_on_root_level_dict_values():
     timestamp = datetime.utcnow()
     key_value_pairs_with_dict_values = [
         ('howdy', None),
-        ('nest3', OrderedDict([('key31', 2.0), ('key32', 0)])),
+        ('empty', {}),
+        ('nest3', OrderedDict([('k31', 2.0), ('k32', 0)])),
         ('nest4', {'y': 3, 'x': 1}),
         ('moarStuff', '!')
     ]
 
     qs_row = serialize(identifier, timestamp, key_value_pairs_with_dict_values)
 
-    assert ',nest3={"key31":2.0,"key32":0},nest4={"y":3,"x":1},moarSt' in qs_row
+    assert ',empty={},nest3={"k31":2.0,"k32":0},nest4={"y":3,"x":1},m' in qs_row
 
 
 def test_each_output_records_ends_with_newline():