Repository: dandi/dandi-cli

Issue:
dandi ls -f json_pp doesn't produce a valid json
we wrapping into a list. ATM `-f json` produces legit "json lines" having each record on a separate line.  But I think we better make it into a  explicit `json_lines` or alike.

`json_pp` takes those lines and renders them across multiple lines thus making overall illegit.  For both `json` and `json_pp` let's place all the records into a list and output that one

note: -f yaml seems to do the right thing and makes a list

Comments:

Patch:
diff --git a/dandi/cli/cmd_ls.py b/dandi/cli/cmd_ls.py
index 4ca3fa06..e0196ac4 100644
--- a/dandi/cli/cmd_ls.py
+++ b/dandi/cli/cmd_ls.py
@@ -33,7 +33,7 @@ List .nwb files and dandisets metadata.
     "--format",
     help="Choose the format/frontend for output. If 'auto', 'pyout' will be "
     "used in case of multiple files, and 'yaml' for a single file.",
-    type=click.Choice(["auto", "pyout", "json", "json_pp", "yaml"]),
+    type=click.Choice(["auto", "pyout", "json", "json_pp", "json_lines", "yaml"]),
     default="auto",
 )
 @click.option(
@@ -66,7 +66,12 @@ def ls(paths, schema, metadata, fields=None, format="auto", recursive=False, job
     """ List .nwb files and dandisets metadata. """
 
     # TODO: more logical ordering in case of fields = None
-    from .formatter import JSONFormatter, PYOUTFormatter, YAMLFormatter
+    from .formatter import (
+        JSONFormatter,
+        JSONLinesFormatter,
+        PYOUTFormatter,
+        YAMLFormatter,
+    )
     from ..consts import metadata_all_fields
 
     # TODO: avoid
@@ -144,6 +149,8 @@ def ls(paths, schema, metadata, fields=None, format="auto", recursive=False, job
         out = JSONFormatter()
     elif format == "json_pp":
         out = JSONFormatter(indent=2)
+    elif format == "json_lines":
+        out = JSONLinesFormatter()
     elif format == "yaml":
         out = YAMLFormatter()
     else:
diff --git a/dandi/cli/formatter.py b/dandi/cli/formatter.py
index 712a32dd..1e831050 100644
--- a/dandi/cli/formatter.py
+++ b/dandi/cli/formatter.py
@@ -22,21 +22,57 @@ class JSONFormatter(Formatter):
     def __init__(self, indent=None, out=None):
         self.out = out or sys.stdout
         self.indent = indent
+        self.first = True
 
     @staticmethod
     def _serializer(o):
         if isinstance(o, datetime.datetime):
-            return o.__str__()
+            return str(o)
+        return o
+
+    def __enter__(self):
+        print("[", end="", file=self.out)
+
+    def __exit__(self, exc_type, exc_value, traceback):
+        if not self.first:
+            print(file=self.out)
+        print("]", file=self.out)
+
+    def __call__(self, rec):
+        import json
+        from textwrap import indent
+
+        if self.first:
+            print(file=self.out)
+            self.first = False
+        else:
+            print(",", file=self.out)
+
+        s = json.dumps(
+            rec, indent=self.indent, sort_keys=True, default=self._serializer
+        )
+        print(indent(s, " " * (self.indent or 2)), end="", file=self.out)
+
+
+class JSONLinesFormatter(Formatter):
+    def __init__(self, indent=None, out=None):
+        self.out = out or sys.stdout
+        self.indent = indent
+
+    @staticmethod
+    def _serializer(o):
+        if isinstance(o, datetime.datetime):
+            return str(o)
         return o
 
     def __call__(self, rec):
         import json
 
-        self.out.write(
+        print(
             json.dumps(
                 rec, indent=self.indent, sort_keys=True, default=self._serializer
-            )
-            + "\n"
+            ),
+            file=self.out,
         )
 
 


Test patch:
diff --git a/dandi/cli/tests/test_formatter.py b/dandi/cli/tests/test_formatter.py
new file mode 100644
index 00000000..1493b9d1
--- /dev/null
+++ b/dandi/cli/tests/test_formatter.py
@@ -0,0 +1,67 @@
+from io import StringIO
+
+import pytest
+
+from ..formatter import JSONFormatter, JSONLinesFormatter
+
+
+def test_json_formatter():
+    out = StringIO()
+    fmtr = JSONFormatter(out=out)
+    with fmtr:
+        fmtr({"foo": 23, "bar": 42})
+        fmtr({"bar": "gnusto", "foo": "cleesh"})
+    assert out.getvalue() == (
+        "[\n"
+        '  {"bar": 42, "foo": 23},\n'
+        '  {"bar": "gnusto", "foo": "cleesh"}\n'
+        "]\n"
+    )
+
+
+def test_json_formatter_indented():
+    out = StringIO()
+    fmtr = JSONFormatter(indent=2, out=out)
+    with fmtr:
+        fmtr({"foo": 23, "bar": 42})
+        fmtr({"bar": "gnusto", "foo": "cleesh"})
+    assert out.getvalue() == (
+        "[\n"
+        "  {\n"
+        '    "bar": 42,\n'
+        '    "foo": 23\n'
+        "  },\n"
+        "  {\n"
+        '    "bar": "gnusto",\n'
+        '    "foo": "cleesh"\n'
+        "  }\n"
+        "]\n"
+    )
+
+
+@pytest.mark.parametrize("indent", [None, 2])
+def test_json_formatter_empty(indent):
+    out = StringIO()
+    fmtr = JSONFormatter(indent=indent, out=out)
+    with fmtr:
+        pass
+    assert out.getvalue() == "[]\n"
+
+
+def test_json_lines_formatter():
+    out = StringIO()
+    fmtr = JSONLinesFormatter(out=out)
+    with fmtr:
+        fmtr({"foo": 23, "bar": 42})
+        fmtr({"bar": "gnusto", "foo": "cleesh"})
+    assert out.getvalue() == (
+        '{"bar": 42, "foo": 23}\n{"bar": "gnusto", "foo": "cleesh"}\n'
+    )
+
+
+def test_json_lines_formatter_empty():
+    out = StringIO()
+    fmtr = JSONLinesFormatter(out=out)
+    with fmtr:
+        pass
+    assert out.getvalue() == ""
diff --git a/dandi/cli/tests/test_ls.py b/dandi/cli/tests/test_ls.py
index 205147ae..6167e55a 100644
--- a/dandi/cli/tests/test_ls.py
+++ b/dandi/cli/tests/test_ls.py
@@ -1,3 +1,5 @@
+import json
+
 from click.testing import CliRunner
 import pytest
 
@@ -5,7 +7,9 @@ from ..command import ls
 from ...utils import yaml_load
 
 
-@pytest.mark.parametrize("format", ("auto", "json", "json_pp", "yaml", "pyout"))
+@pytest.mark.parametrize(
+    "format", ("auto", "json", "json_pp", "json_lines", "yaml", "pyout")
+)
 def test_smoke(simple1_nwb_metadata, simple1_nwb, format):
     runner = CliRunner()
     r = runner.invoke(ls, ["-f", format, simple1_nwb])
@@ -13,10 +17,15 @@ def test_smoke(simple1_nwb_metadata, simple1_nwb, format):
     # we would need to redirect pyout for its analysis
     out = r.stdout
 
-    if format.startswith("json"):
-        import json
-
+    if format == "json_lines":
         load = json.loads
+    elif format.startswith("json"):
+
+        def load(s):
+            obj = json.loads(s)
+            assert len(obj) == 1  # will be a list with a single elem
+            return obj[0]
+
     elif format == "yaml":
 
         def load(s):