Repository: pybel/pybel

Issue:
Streamable graphs
Implement an edge container class:

- [ ] I/O to and from JSON so a graph can be easily
- [ ] Serialization and deserialization from JSONL
- [ ] add to BEL graph functionality
- [ ] create from (u,v,k,d)

Discussion:
codecov[bot]: # [Codecov](https://codecov.io/gh/pybel/pybel/pull/451?src=pr&el=h1) Report
> Merging [#451](https://codecov.io/gh/pybel/pybel/pull/451?src=pr&el=desc) into [master](https://codecov.io/gh/pybel/pybel/commit/5edc87af715f3ab2808735bbff72e774846144d4&el=desc) will **decrease** coverage by `0.17%`.
> The diff coverage is `50.00%`.

[![Impacted file tree graph](https://codecov.io/gh/pybel/pybel/pull/451/graphs/tree.svg?width=650&height=150&src=pr&token=J7joRTRygG)](https://codecov.io/gh/pybel/pybel/pull/451?src=pr&el=tree)

```diff
@@            Coverage Diff             @@
##           master     #451      +/-   ##
==========================================
- Coverage   79.04%   78.86%   -0.18%     
==========================================
  Files         171      172       +1     
  Lines        8937     8990      +53     
  Branches     1290     1298       +8     
==========================================
+ Hits         7064     7090      +26     
- Misses       1595     1622      +27     
  Partials      278      278              
```


| [Impacted Files](https://codecov.io/gh/pybel/pybel/pull/451?src=pr&el=tree) | Coverage Δ | |
|---|---|---|
| [src/pybel/io/sbel.py](https://codecov.io/gh/pybel/pybel/pull/451/diff?src=pr&el=tree#diff-c3JjL3B5YmVsL2lvL3NiZWwucHk=) | `44.89% <44.89%> (ø)` | |
| [src/pybel/\_\_init\_\_.py](https://codecov.io/gh/pybel/pybel/pull/451/diff?src=pr&el=tree#diff-c3JjL3B5YmVsL19faW5pdF9fLnB5) | `100.00% <100.00%> (ø)` | |
| [src/pybel/io/\_\_init\_\_.py](https://codecov.io/gh/pybel/pybel/pull/451/diff?src=pr&el=tree#diff-c3JjL3B5YmVsL2lvL19faW5pdF9fLnB5) | `100.00% <100.00%> (ø)` | |
| [src/pybel/io/nodelink.py](https://codecov.io/gh/pybel/pybel/pull/451/diff?src=pr&el=tree#diff-c3JjL3B5YmVsL2lvL25vZGVsaW5rLnB5) | `82.22% <100.00%> (+0.40%)` | :arrow_up: |

------

[Continue to review full report at Codecov](https://codecov.io/gh/pybel/pybel/pull/451?src=pr&el=continue).
> **Legend** - [Click here to learn more](https://docs.codecov.io/docs/codecov-delta)
> `Δ = absolute <relative> (impact)`, `ø = not affected`, `? = missing data`
> Powered by [Codecov](https://codecov.io/gh/pybel/pybel/pull/451?src=pr&el=footer). Last update [5edc87a...ea76fa5](https://codecov.io/gh/pybel/pybel/pull/451?src=pr&el=lastupdated). Read the [comment docs](https://docs.codecov.io/docs/pull-request-comments).


Patch:
diff --git a/docs/source/reference/io.rst b/docs/source/reference/io.rst
index 39037c78..0cda191a 100644
--- a/docs/source/reference/io.rst
+++ b/docs/source/reference/io.rst
@@ -78,6 +78,19 @@ Node-Link JSON
 .. autofunction:: pybel.from_nodelink_gz
 .. autofunction:: pybel.to_nodelink_gz
 
+Streamable BEL (JSONL)
+~~~~~~~~~~~~~~~~~~~~~~
+.. automodule:: pybel.io.sbel
+
+.. autofunction:: pybel.from_sbel
+.. autofunction:: pybel.to_sbel
+
+.. autofunction:: pybel.from_sbel_file
+.. autofunction:: pybel.to_sbel_file
+
+.. autofunction:: pybel.from_sbel_gz
+.. autofunction:: pybel.to_sbel_gz
+
 Cyberinfrastructure Exchange
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~
 .. automodule:: pybel.io.cx
diff --git a/setup.cfg b/setup.cfg
index 8e194350..6ad9e825 100644
--- a/setup.cfg
+++ b/setup.cfg
@@ -101,6 +101,8 @@ pybel.importer =
     bel.gz                = pybel.io.lines:from_bel_script_gz
     bel.nodelink.json     = pybel.io.nodelink:from_nodelink_file
     bel.nodelink.json.gz  = pybel.io.nodelink:from_nodelink_gz
+    bel.jsonl             = pybel.io.sbel:from_sbel_file
+    bel.jsonl.gz          = pybel.io.sbel:from_sbel_gz
     bel.cx.json           = pybel.io.cx:from_cx_file
     bel.cx.json.gz        = pybel.io.cx:from_cx_gz
     bel.graphdati.json    = pybel.io.graphdati:from_graphdati_file
@@ -117,6 +119,8 @@ pybel.exporter =
     bel.gz                = pybel.canonicalize:to_bel_script_gz
     bel.nodelink.json     = pybel.io.nodelink:to_nodelink_file
     bel.nodelink.json.gz  = pybel.io.nodelink:to_nodelink_gz
+    bel.jsonl             = pybel.io.sbel:to_sbel_file
+    bel.jsonl.gz          = pybel.io.sbel:to_sbel_gz
     bel.cx.json           = pybel.io.cx:to_cx_file
     bel.cx.json.gz        = pybel.io.cx:to_cx_gz
     bel.graphdati.json    = pybel.io.graphdati:to_graphdati_file
diff --git a/src/pybel/__init__.py b/src/pybel/__init__.py
index 2bac9e5f..120bed63 100644
--- a/src/pybel/__init__.py
+++ b/src/pybel/__init__.py
@@ -36,6 +36,7 @@ from .io.nodelink import (
     to_nodelink_file, to_nodelink_gz, to_nodelink_jsons,
 )
 from .io.pynpa import to_npa_dfs, to_npa_directory
+from .io.sbel import from_sbel, from_sbel_file, from_sbel_gz, to_sbel, to_sbel_file, to_sbel_gz
 from .io.spia import to_spia_dfs, to_spia_excel, to_spia_tsvs
 from .io.tsv import to_edgelist, to_tsv
 from .io.umbrella_nodelink import to_umbrella_nodelink, to_umbrella_nodelink_file, to_umbrella_nodelink_gz
diff --git a/src/pybel/io/__init__.py b/src/pybel/io/__init__.py
index a146e1ea..711b05c7 100644
--- a/src/pybel/io/__init__.py
+++ b/src/pybel/io/__init__.py
@@ -39,6 +39,7 @@ from .nodelink import (
     to_nodelink_file, to_nodelink_gz, to_nodelink_jsons,
 )
 from .pynpa import to_npa_dfs, to_npa_directory
+from .sbel import from_sbel, from_sbel_file, from_sbel_gz, to_sbel, to_sbel_file, to_sbel_gz
 from .spia import to_spia_dfs, to_spia_excel, to_spia_tsvs
 from .tsv import to_edgelist, to_tsv
 from .umbrella_nodelink import to_umbrella_nodelink, to_umbrella_nodelink_file, to_umbrella_nodelink_gz
diff --git a/src/pybel/io/nodelink.py b/src/pybel/io/nodelink.py
index 19f9f7a2..a8ad5ad3 100644
--- a/src/pybel/io/nodelink.py
+++ b/src/pybel/io/nodelink.py
@@ -146,6 +146,13 @@ def _augment_node(node: BaseEntity) -> BaseEntity:
     return rv
 
 
+def _fix_annotation_list(graph: BELGraph):
+    graph.graph[GRAPH_ANNOTATION_LIST] = {
+        keyword: set(values)
+        for keyword, values in graph.graph.get(GRAPH_ANNOTATION_LIST, {}).items()
+    }
+
+
 def _from_nodelink_json_helper(data: Mapping[str, Any]) -> BELGraph:
     """Return graph from node-link data format.
 
@@ -153,10 +160,7 @@ def _from_nodelink_json_helper(data: Mapping[str, Any]) -> BELGraph:
     """
     graph = BELGraph()
     graph.graph = data.get('graph', {})
-    graph.graph[GRAPH_ANNOTATION_LIST] = {
-        keyword: set(values)
-        for keyword, values in graph.graph.get(GRAPH_ANNOTATION_LIST, {}).items()
-    }
+    _fix_annotation_list(graph)
 
     mapping = []
 
diff --git a/src/pybel/io/sbel.py b/src/pybel/io/sbel.py
new file mode 100644
index 00000000..386f62a6
--- /dev/null
+++ b/src/pybel/io/sbel.py
@@ -0,0 +1,122 @@
+# -*- coding: utf-8 -*-
+
+"""Streamable BEL as JSON."""
+
+import gzip
+import json
+from typing import Any, Iterable, List, TextIO, Union
+
+from networkx.utils import open_file
+
+from .nodelink import _augment_node, _fix_annotation_list
+from ..constants import OBJECT, SUBJECT
+from ..struct.graph import BELGraph, _handle_modifier
+from ..tokens import parse_result_to_dsl
+from ..utils import hash_edge
+
+__all__ = [
+    'to_sbel_file',
+    'to_sbel',
+    'to_sbel_gz',
+    'from_sbel',
+    'from_sbel_gz',
+    'from_sbel_file',
+]
+
+SBEL = Any
+
+
+@open_file(1, mode='w')
+def to_sbel_file(graph: BELGraph, path: Union[str, TextIO], separators=(',', ':'), **kwargs) -> None:
+    """Write this graph as BEL JSONL to a file.
+
+    :param graph: A BEL graph
+    :param separators: The separators used in :func:`json.dumps`
+    :param path: A path or file-like
+    """
+    for i in iterate_sbel(graph):
+        print(json.dumps(i, ensure_ascii=False, separators=separators, **kwargs), file=path)
+
+
+def to_sbel_gz(graph: BELGraph, path: str, separators=(',', ':'), **kwargs) -> None:
+    """Write a graph as BEL JSONL to a gzip file.
+
+    :param graph: A BEL graph
+    :param separators: The separators used in :func:`json.dumps`
+    :param path: A path for a gzip file
+    """
+    with gzip.open(path, 'wt') as file:
+        to_sbel_file(graph, file, separators=separators, **kwargs)
+
+
+def to_sbel(graph: BELGraph) -> List[SBEL]:
+    """Create a list of JSON dictionaries corresponding to lines in BEL JSONL."""
+    return list(iterate_sbel(graph))
+
+
+def iterate_sbel(graph: BELGraph) -> Iterable[SBEL]:
+    """Iterate over JSON dictionaries corresponding to lines in BEL JSONL."""
+    yield graph.graph.copy()
+    for u, v, k, d in graph.edges(data=True, keys=True):
+        yield {
+            'source': _augment_node(u),
+            'target': _augment_node(v),
+            'key': k,
+            **d,
+        }
+
+
+def from_sbel(it: Iterable[SBEL], includes_metadata: bool = True) -> BELGraph:
+    """Load a BEL graph from an iterable of dictionaries corresponding to lines in BEL JSONL.
+
+    :param it: An iterable of dictionaries.
+    :param includes_metadata: By default, interprets the first element of the iterable as the graph's metadata.
+     Switch to ``False`` to disable.
+    :return: A BEL graph
+    """
+    it = iter(it)
+    rv = BELGraph()
+    if includes_metadata:
+        rv.graph.update(next(it))
+        _fix_annotation_list(rv)
+    add_sbel(rv, it)
+    return rv
+
+
+def add_sbel(graph: BELGraph, it: Iterable[SBEL]) -> None:
+    """Add dictionaries to a BEL graph.
+
+    :param graph: A BEL graph
+    :param it: An iterable of dictionaries.
+    """
+    for data in it:
+        u = parse_result_to_dsl(data['source'])
+        v = parse_result_to_dsl(data['target'])
+        edge_data = {
+            k: v
+            for k, v in data.items()
+            if k not in {'source', 'target', 'key'}
+        }
+        for side in (SUBJECT, OBJECT):
+            side_data = edge_data.get(side)
+            if side_data:
+                _handle_modifier(side_data)
+        graph.add_edge(u, v, key=hash_edge(u, v, edge_data), **edge_data)
+
+
+@open_file(0, mode='r')
+def from_sbel_file(path: Union[str, TextIO]) -> BELGraph:
+    """Build a graph from the BEL JSONL contained in the given file.
+
+    :param path: A path or file-like
+    """
+    return from_sbel((
+        json.loads(line)
+        for line in path
+    ))
+
+
+def from_sbel_gz(path: str) -> BELGraph:
+    """Read a graph as BEL JSONL from a gzip file."""
+    with gzip.open(path, 'rt') as file:
+        return from_sbel_file(file)


Test patch:
diff --git a/tests/test_io/test_import.py b/tests/test_io/test_import.py
index c7986375..f7a87a65 100644
--- a/tests/test_io/test_import.py
+++ b/tests/test_io/test_import.py
@@ -10,6 +10,7 @@ import unittest
 from io import BytesIO, StringIO
 from pathlib import Path
 
+import pybel
 from pybel import (
     BELGraph, from_bel_script, from_bel_script_url, from_bytes, from_nodelink, from_nodelink_file, from_pickle,
     to_bel_script_lines, to_bytes, to_csv, to_graphml, to_gsea, to_nodelink, to_nodelink_file, to_pickle, to_sif,
@@ -96,6 +97,28 @@ class TestExampleInterchange(unittest.TestCase):
         graph = from_nodelink_file(sio)
         self._help_test_equal(graph)
 
+    def test_thorough_sbel(self):
+        """Test the round-trip through SBEL."""
+        s = pybel.to_sbel(sialic_acid_graph)
+        graph = pybel.from_sbel(s)
+        self._help_test_equal(graph)
+
+    def test_thorough_sbel_file(self):
+        """Test the round-trip through a SBEL file."""
+        sio = StringIO()
+        pybel.to_sbel_file(sialic_acid_graph, sio)
+        sio.seek(0)
+        graph = pybel.from_sbel_file(sio)
+        self._help_test_equal(graph)
+
+    def test_thorough_sbel_gzip_path(self):
+        """Test round trip through a SBEL gzipped file."""
+        with tempfile.TemporaryDirectory() as directory:
+            path = os.path.join(directory, 'test.gzip')
+            pybel.to_sbel_gz(sialic_acid_graph, path)
+            graph = pybel.from_sbel_gz(path)
+        self._help_test_equal(graph)
+
 
 class TestInterchange(TemporaryCacheClsMixin, BelReconstitutionMixin):
     @classmethod