Repository: cogent3/cogent3
Pull request #1243 (2022-12-19 22:04:35)

MAINT: testing and fixing corner cases


Patch:
diff --git a/src/cogent3/app/data_store_new.py b/src/cogent3/app/data_store_new.py
index 33d17d9e6..6ab700cf1 100644
--- a/src/cogent3/app/data_store_new.py
+++ b/src/cogent3/app/data_store_new.py
@@ -40,6 +40,7 @@ _LOG_TABLE = "logs"
 _MD5_TABLE = "md5"
 
 StrOrBytes = Union[str, bytes]
+NoneType = type(None)
 
 
 class Mode(Enum):
@@ -216,7 +217,7 @@ class DataStoreABC(ABC):
                     data["python"],
                     data["user"],
                     data["command_string"],
-                    data["composable function"],
+                    data.get("composable function", ""),
                 ]
             )
             rows.append(row)
@@ -283,25 +284,32 @@ class DataStoreABC(ABC):
 
     def validate(self) -> TabularType:
         correct_md5 = len(self.members)
+        missing_md5 = 0
         for m in self.members:
             data = m.read()
             md5 = self.md5(m.unique_id)
-            if md5 != get_text_hexdigest(data):
+            if md5 is None:
+                missing_md5 += 1
                 correct_md5 -= 1
-        incorrect_md5 = len(self.members) - correct_md5
+            elif md5 != get_text_hexdigest(data):
+                correct_md5 -= 1
+
+        incorrect_md5 = len(self.members) - correct_md5 - missing_md5
 
         return Table(
             header=["Condition", "Value"],
             data=[
                 ["Num md5sum correct", correct_md5],
                 ["Num md5sum incorrect", incorrect_md5],
+                ["Num md5sum missing", missing_md5],
                 ["Has log", len(self.logs) > 0],
             ],
             title="validate status",
             index_name="Condition",
         )
 
-    def md5(self, unique_id: str) -> str:
+    @abstractmethod
+    def md5(self, unique_id: str) -> Union[str, NoneType]:
         """
         Parameters
         ----------
@@ -312,11 +320,6 @@ class DataStoreABC(ABC):
         -------
         md5 checksum for the member, if available, None otherwise
         """
-        unique_id = Path(unique_id)
-        unique_id = re.sub(rf"[.]({self.suffix}|json)$", ".txt", unique_id.name)
-        path = self.source / _MD5_TABLE / unique_id
-
-        return path.read_text() if path.exists() else None
 
 
 class DataMember(DataMemberABC):
@@ -501,6 +504,23 @@ class DataStoreDirectory(DataStoreABC):
     def write_log(self, *, unique_id: str, data: str) -> None:
         _ = self._write(subdir=_LOG_TABLE, unique_id=unique_id, suffix="log", data=data)
 
+    def md5(self, unique_id: str) -> Union[str, NoneType]:
+        """
+        Parameters
+        ----------
+        unique_id
+            name of data store member
+
+        Returns
+        -------
+        md5 checksum for the member, if available, None otherwise
+        """
+        unique_id = Path(unique_id)
+        unique_id = re.sub(rf"[.]({self.suffix}|json)$", ".txt", unique_id.name)
+        path = self.source / _MD5_TABLE / unique_id
+
+        return path.read_text() if path.exists() else None
+
 
 @singledispatch
 def get_data_source(data) -> str:
diff --git a/src/cogent3/app/sqlite_data_store.py b/src/cogent3/app/sqlite_data_store.py
index 0399c07e4..940360b8e 100644
--- a/src/cogent3/app/sqlite_data_store.py
+++ b/src/cogent3/app/sqlite_data_store.py
@@ -36,7 +36,7 @@ __status__ = "Alpha"
 _RESULT_TABLE = "results"
 _MEMORY = ":memory:"
 _mem_pattern = re.compile(r"^\s*[:]{0,1}memory[:]{0,1}\s*$")
-
+NoneType = type(None)
 
 # create db
 def open_sqlite_db_rw(path: Union[str, Path]):
@@ -254,10 +254,12 @@ class DataStoreSqlite(DataStoreABC):
         """
         if self._log_id is None:
             self._init_log()
+
         if table_name == _LOG_TABLE:
             cmnd = f"UPDATE {table_name} SET data =?, log_name =? WHERE log_id=?"
             self.db.execute(cmnd, (data, unique_id, self._log_id))
             return None
+
         md5 = get_text_hexdigest(data)
 
         if unique_id in self and self.mode is not APPEND:
@@ -266,6 +268,7 @@ class DataStoreSqlite(DataStoreABC):
         else:
             cmnd = f"INSERT INTO {table_name} (record_id,data,log_id,md5,is_completed) VALUES (?,?,?,?,?)"
             self.db.execute(cmnd, (unique_id, data, self._log_id, md5, is_completed))
+
         return DataMember(data_store=self, unique_id=unique_id)
 
     def drop_not_completed(self) -> None:
@@ -362,7 +365,7 @@ class DataStoreSqlite(DataStoreABC):
             self._not_completed.append(member)
         return member
 
-    def md5(self, unique_id: str) -> str:  # we have it in base class
+    def md5(self, unique_id: str) -> Union[str, NoneType]:  # we have it in base class
         """
         Parameters
         ----------
@@ -372,13 +375,10 @@ class DataStoreSqlite(DataStoreABC):
         -------
         md5 checksum for the member, if available, None otherwise
         """
-        unique_id = Path(unique_id)
-        unique_id = re.sub(rf"[.]({self.store_suffix}|json)$", ".txt", unique_id.name)
-
         cmnd = f"SELECT * FROM {_RESULT_TABLE} WHERE record_id = ?"
         result = self.db.execute(cmnd, (unique_id,)).fetchone()
 
-        return result["md5"]
+        return result["md5"] if result else None
 
     @property
     def describe(self):


Test patch:
diff --git a/tests/test_app/test_composable.py b/tests/test_app/test_composable.py
index d5096129f..b942b56c3 100644
--- a/tests/test_app/test_composable.py
+++ b/tests/test_app/test_composable.py
@@ -1163,6 +1163,28 @@ def test_complex_type_allowed_depths(hint):
     __app_registry.pop(get_object_provenance(x), None)
 
 
+def test_apply_to_only_appends(half_dstore1, half_dstore2):
+    half_dstore1._mode = APPEND
+    reader1 = io_app.load_aligned(format="fasta", moltype="dna")
+    min_length1 = sample_app.min_length(10)
+    writer1 = io_app_new.write_seqs(half_dstore1)
+    process1 = reader1 + min_length1 + writer1
+    # create paths as strings
+    dstore1 = open_data_store(half_dstore1.source, suffix="fasta")
+    dstore1 = [str(Path(m.data_store.source) / m.unique_id) for m in dstore1]
+    # check fail on append the same records
+    with pytest.raises(IOError):
+        _ = process1.apply_to(dstore1, id_from_source=get_data_source)
+
+    half_dstore2._mode = APPEND
+    reader2 = io_app.load_aligned(format="fasta", moltype="dna")
+    min_length2 = sample_app.min_length(10)
+    writer2 = io_app_new.write_seqs(half_dstore2)
+    process2 = reader2 + min_length2 + writer2
+    # check not fail on append new records
+    _ = process2.apply_to(dstore1, id_from_source=get_data_source)
+
+
 def test_skip_not_completed():
     @define_app(skip_not_completed=False)
     def takes_not_completed(val: c3types.SerialisableType) -> dict:
diff --git a/tests/test_app/test_data_store_new.py b/tests/test_app/test_data_store_new.py
index 0415d7fc1..ba4e35442 100644
--- a/tests/test_app/test_data_store_new.py
+++ b/tests/test_app/test_data_store_new.py
@@ -445,6 +445,25 @@ def test_md5_sum(nc_dstore):
         assert md5 == get_text_hexdigest(data)
 
 
+def test_md5_none(fasta_dir):
+    dstore = DataStoreDirectory(fasta_dir, suffix="fasta")
+    for m in dstore.members:
+        assert m.md5 is None
+
+
+def test_md5_missing(nc_dstore):
+    nc_dstore.md5("unknown") is None
+
+
+def test_summary_logs_missing_field(nc_dstore):
+    log_path = Path(nc_dstore.source) / nc_dstore.logs[0].unique_id
+    data = [
+        l for l in log_path.read_text().splitlines() if "composable function" not in l
+    ]
+    log_path.write_text("\n".join(data))
+    assert isinstance(nc_dstore.summary_logs, Table)
+
+
 def test_summary_logs(full_dstore):
     # log summary has a row per log file and a column for each property
     got = full_dstore.summary_logs
diff --git a/tests/test_app/test_sqlite_data_store.py b/tests/test_app/test_sqlite_data_store.py
index 932047f14..6ee418612 100644
--- a/tests/test_app/test_sqlite_data_store.py
+++ b/tests/test_app/test_sqlite_data_store.py
@@ -414,7 +414,7 @@ def test_limit_datastore(full_dstore_sqlite):
 
 def test_validate(full_dstore_sqlite):
     r = full_dstore_sqlite.validate()
-    assert r.shape == (3, 2)
+    assert r.shape == (4, 2)
 
 
 def test_no_not_completed(sql_dstore):
@@ -516,3 +516,28 @@ def test_lock_firsttime(full_dstore_sqlite):
 
 def test_db_without_logs(ro_sql_dstore):
     assert len(ro_sql_dstore.logs) == 0
+
+
+@pytest.fixture(scope="function")
+def md5_none(full_dstore_sqlite):
+    """create a data store with empty md5 fields"""
+    full_dstore_sqlite.db.execute(
+        "UPDATE results SET md5=? WHERE record_id LIKE '%'", (None,)
+    )
+    return full_dstore_sqlite
+
+
+def test_md5_none(md5_none):
+    m = md5_none[0]
+    assert m.md5 is None
+
+
+def test_md5_missing(md5_none):
+    md5_none.md5("unknown")
+
+
+def test_validate_missing_md5(md5_none):
+    t = md5_none.validate()
+    assert t["Num md5sum missing", "Value"] == 9
+    for c in ("correct", "incorrect"):
+        assert t[f"Num md5sum {c}", "Value"] == 0