Repository: cogent3/cogent3
Pull request #1276 (2023-01-18 07:56:55)

ENH: new standalone summary_not_completeds function
[NEW] in data_store_new, adds an argument for deserialising which allows users
    a way to deal with records stored in binary format


Patch:
diff --git a/src/cogent3/app/data_store_new.py b/src/cogent3/app/data_store_new.py
index 7aee5da1e..78fbc8f04 100644
--- a/src/cogent3/app/data_store_new.py
+++ b/src/cogent3/app/data_store_new.py
@@ -232,47 +232,7 @@ class DataStoreABC(ABC):
     def summary_not_completed(self) -> TabularType:
         """returns a table summarising not completed results"""
         # detect last exception line
-        err_pat = re.compile(r"[A-Z][a-z]+[A-Z][a-z]+\:.+")
-        types = defaultdict(list)
-        indices = "type", "origin"
-        num_bytes = 0
-        for member in self.not_completed:
-            record = member.read()
-            if isinstance(record, bytes):
-                num_bytes += 1
-                continue
-            record = deserialise_object(record)
-            key = tuple(getattr(record, k, None) for k in indices)
-            match = err_pat.findall(record.message)
-            types[key].append([match[-1] if match else record.message, record.source])
-
-        header = list(indices) + ["message", "num", "source"]
-        if num_bytes == len(self.not_completed):
-            return Table(
-                header=header,
-                title="Cannot summarise not_completed as they are all bytes, "
-                "use an appropriate reader",
-            )
-
-        rows = []
-        maxtring = reprlib.aRepr.maxstring
-        reprlib.aRepr.maxstring = 45
-
-        for record in types:
-            messages, sources = list(zip(*types[record]))
-            messages = reprlib.repr(
-                ", ".join(m.splitlines()[-1] for m in set(messages))
-            )
-            sources = reprlib.repr(", ".join(s.splitlines()[-1] for s in sources))
-            row = list(record) + [
-                messages,
-                len(types[record]),
-                sources,
-            ]
-            rows.append(row)
-
-        reprlib.aRepr.maxstring = maxtring  # restoring original val
-        return Table(header=header, data=rows, title="not completed records")
+        return summary_not_completeds(self.not_completed)
 
     @property
     def describe(self) -> TabularType:
@@ -351,6 +311,57 @@ class DataMember(DataMemberABC):
         return self._unique_id
 
 
+def summary_not_completeds(
+    not_completed: list[DataMemberABC], deserialise: Optional[callable] = None
+) -> Table:
+    """
+    Parameters
+    ----------
+    not_completed
+        list of DataMember instances for notcompleted records
+    deserialise
+        a callable for converting not completed contents, the result of member.read() must be a json string
+    """
+    err_pat = re.compile(r"[A-Z][a-z]+[A-Z][a-z]+\:.+")
+    types = defaultdict(list)
+    indices = "type", "origin"
+    num_bytes = 0
+    for member in not_completed:
+        record = member.read()
+        if deserialise:
+            record = deserialise(record)
+        if isinstance(record, bytes):
+            num_bytes += 1
+            continue
+        record = deserialise_object(record)
+        key = tuple(getattr(record, k, None) for k in indices)
+        match = err_pat.findall(record.message)
+        types[key].append([match[-1] if match else record.message, record.source])
+    header = list(indices) + ["message", "num", "source"]
+    if num_bytes == len(not_completed):
+        return Table(
+            header=header,
+            title="Cannot summarise not_completed as they are all bytes, "
+            "use an appropriate reader",
+        )
+
+    rows = []
+    maxtring = reprlib.aRepr.maxstring
+    reprlib.aRepr.maxstring = 45
+    for record in types:
+        messages, sources = list(zip(*types[record]))
+        messages = reprlib.repr(", ".join(m.splitlines()[-1] for m in set(messages)))
+        sources = reprlib.repr(", ".join(s.splitlines()[-1] for s in sources))
+        row = list(record) + [
+            messages,
+            len(types[record]),
+            sources,
+        ]
+        rows.append(row)
+    reprlib.aRepr.maxstring = maxtring  # restoring original val
+    return Table(header=header, data=rows, title="not completed records")
+
+
 class DataStoreDirectory(DataStoreABC):
     def __init__(
         self,


Test patch:
diff --git a/tests/test_app/test_data_store_new.py b/tests/test_app/test_data_store_new.py
index 911ffa6dc..bb42f7aab 100644
--- a/tests/test_app/test_data_store_new.py
+++ b/tests/test_app/test_data_store_new.py
@@ -25,6 +25,7 @@ from cogent3.app.data_store_new import (
     get_data_source,
     get_unique_id,
     load_record_from_json,
+    summary_not_completeds,
 )
 from cogent3.util.table import Table
 from cogent3.util.union_dict import UnionDict
@@ -480,6 +481,22 @@ def test_summary_not_completed(full_dstore):
     assert isinstance(got, Table)
 
 
+@pytest.mark.parametrize("use_dser", (False, True))
+def test_summary_not_completed_func(nc_objects, use_dser):
+    dstore = io_app.open_data_store(":memory:", mode="w")
+    writer = io_app.write_db(dstore)
+    deser = io_app.load_db().deserialiser if use_dser else None
+    for nc in nc_objects:
+        writer(nc)
+
+    got = summary_not_completeds(dstore, deserialise=deser)
+    assert isinstance(got, Table)
+    if use_dser:
+        assert got.shape[0] >= 1
+    else:
+        assert got.shape[0] == 0
+
+
 def test_describe(full_dstore):
     got = full_dstore.describe
     assert got.shape >= (3, 2)