Repository: cogent3/cogent3
Pull request #1290 (2023-02-06 07:10:12)

ENH: DataStoreSqlite summary_not_completed now works with defaults


Patch:
diff --git a/src/cogent3/app/data_store_new.py b/src/cogent3/app/data_store_new.py
index 1466381d9..27e414a26 100644
--- a/src/cogent3/app/data_store_new.py
+++ b/src/cogent3/app/data_store_new.py
@@ -348,10 +348,15 @@ def summary_not_completeds(
     rows = []
     maxtring = reprlib.aRepr.maxstring
     reprlib.aRepr.maxstring = 45
+    limit_len = 45
     for record in types:
         messages, sources = list(zip(*types[record]))
         messages = reprlib.repr(", ".join(m.splitlines()[-1] for m in set(messages)))
-        sources = reprlib.repr(", ".join(s.splitlines()[-1] for s in sources if s))
+        sources = ", ".join(s.splitlines()[-1] for s in sources if s)
+        if len(sources) > limit_len:
+            idx = sources.rfind(",", None, limit_len) + 1
+            idx = idx if idx > 0 else limit_len
+            sources = f"{sources[:idx]} ..."
         row = list(record) + [
             messages,
             len(types[record]),
diff --git a/src/cogent3/app/io_new.py b/src/cogent3/app/io_new.py
index 94031ac38..170da78ff 100644
--- a/src/cogent3/app/io_new.py
+++ b/src/cogent3/app/io_new.py
@@ -439,12 +439,15 @@ class load_json:
         return result
 
 
+DEFAULT_DESERIALISER = unpickle_it() + from_primitive()
+
+
 @define_app(app_type=LOADER)
 class load_db:
     """Loads serialised cogent3 objects from a db.
     Returns whatever object type was stored."""
 
-    def __init__(self, deserialiser: callable = unpickle_it() + from_primitive()):
+    def __init__(self, deserialiser: callable = DEFAULT_DESERIALISER):
         self.deserialiser = deserialiser
 
     def main(self, identifier: IdentifierType) -> SerialisableType:
@@ -562,6 +565,9 @@ class write_tabular:
         return self.data_store.write(unique_id=identifier, data=output)
 
 
+DEFAULT_SERIALISER = to_primitive() + pickle_it()
+
+
 @define_app(app_type=WRITER, skip_not_completed=False)
 class write_db:
     """Write serialised objects to a database instance."""
@@ -571,7 +577,7 @@ class write_db:
         self,
         data_store: DataStoreABC,
         id_from_source: callable = get_unique_id,
-        serialiser: callable = to_primitive() + pickle_it(),
+        serialiser: callable = DEFAULT_SERIALISER,
     ):
         """
         serialiser
diff --git a/src/cogent3/app/sqlite_data_store.py b/src/cogent3/app/sqlite_data_store.py
index c97171108..fed705cba 100644
--- a/src/cogent3/app/sqlite_data_store.py
+++ b/src/cogent3/app/sqlite_data_store.py
@@ -11,6 +11,7 @@ from typing import Optional, Union
 
 from scitrack import get_text_hexdigest
 
+from cogent3.app import typing as c3_types
 from cogent3.app.data_store_new import (
     _LOG_TABLE,
     APPEND,
@@ -410,3 +411,13 @@ class DataStoreSqlite(DataStoreABC):
 
         n = get_object_provenance(obj)
         self.db.execute("UPDATE state SET record_type=? WHERE state_id=1", (n,))
+
+    @property
+    def summary_not_completed(self) -> c3_types.TabularType:
+        """returns a table summarising not completed results"""
+        from .data_store_new import summary_not_completeds
+        from .io_new import DEFAULT_DESERIALISER
+
+        return summary_not_completeds(
+            self.not_completed, deserialise=DEFAULT_DESERIALISER
+        )


Test patch:
diff --git a/tests/test_app/test_io_new.py b/tests/test_app/test_io_new.py
index 2222ab962..6d5bf0036 100644
--- a/tests/test_app/test_io_new.py
+++ b/tests/test_app/test_io_new.py
@@ -16,6 +16,7 @@ from cogent3 import DNA, get_app, open_data_store
 from cogent3.app import io_new as io_app
 from cogent3.app.composable import NotCompleted, source_proxy
 from cogent3.app.data_store_new import DataMember, DataStoreDirectory, Mode
+from cogent3.app.io_new import DEFAULT_DESERIALISER, DEFAULT_SERIALISER
 from cogent3.core.alignment import ArrayAlignment, SequenceCollection
 from cogent3.core.profile import PSSM, MotifCountsArray, MotifFreqsArray
 from cogent3.evolve.fast_distance import DistanceMatrix
@@ -578,3 +579,12 @@ def test_open_suffix_dirname(tmp_dir):
     outpath.mkdir(exist_ok=True)
     dstore = open_data_store(outpath, suffix="txt")
     assert isinstance(dstore, DataStoreDirectory)
+
+
+@pytest.mark.parametrize("data", ({"a": [0, 1]}, DNA))
+def test_default_serialiser_deserialiser(data):
+    # the default deserialiser should successfully reverse the
+    # default serialiser
+    s = DEFAULT_SERIALISER(data)
+    ds = DEFAULT_DESERIALISER(s)
+    assert ds == data
diff --git a/tests/test_app/test_sqlite_data_store.py b/tests/test_app/test_sqlite_data_store.py
index a66d631b7..06cb39592 100644
--- a/tests/test_app/test_sqlite_data_store.py
+++ b/tests/test_app/test_sqlite_data_store.py
@@ -343,12 +343,6 @@ def test_summary_logs(full_dstore_sqlite):
     assert isinstance(got, Table)
 
 
-def test_summary_not_completed(full_dstore_sqlite):
-    got = full_dstore_sqlite.summary_not_completed
-    assert got.shape >= (1, 1)
-    assert isinstance(got, Table)
-
-
 def test_no_not_completed_subdir(full_dstore_sqlite):
     expect = f"{len(full_dstore_sqlite.completed)+len(full_dstore_sqlite.not_completed)}x member"
     assert str(full_dstore_sqlite).startswith(expect)
@@ -443,12 +437,6 @@ def test_summary_logs(full_dstore_sqlite):
     assert isinstance(got, Table)
 
 
-def test_summary_not_completed(full_dstore_sqlite):
-    got = full_dstore_sqlite.summary_not_completed
-    assert got.shape >= (1, 1)
-    assert isinstance(got, Table)
-
-
 def test_read_unknown_table(full_dstore_sqlite):
     with pytest.raises(ValueError):
         full_dstore_sqlite.read("unknown_table/id")
@@ -478,12 +466,6 @@ def test_summary_logs(full_dstore_sqlite):
     assert isinstance(got, Table)
 
 
-def test_summary_not_completed(full_dstore_sqlite):
-    got = full_dstore_sqlite.summary_not_completed
-    assert got.shape >= (1, 1)
-    assert isinstance(got, Table)
-
-
 def test_set_record_type(full_dstore_sqlite):
     from cogent3 import make_table
     from cogent3.util.misc import get_object_provenance
@@ -586,3 +568,17 @@ def test_append_makes_logs(tmp_dir, ro_dir_dstore, name, suffix):
     # should be a row for each log
     summary = got2.summary_logs
     assert summary.shape[0] == 2
+
+
+def test_summary_not_completed(nc_objects):
+    dstore = open_data_store(":memory:", mode="w")
+    writer = get_app("write_db", dstore)
+    for nc in nc_objects.values():
+        writer(nc)
+
+    # relying on the fact that all nc_objects have same origin
+    # and message, so those columns can be readily interrogated
+    summary = dstore.summary_not_completed
+    vals = summary.tolist(columns=["origin", "message", "num"])
+    assert len(vals) == 1
+    assert vals[0] == ["location", "'message'", 3]