Repository: cogent3/cogent3
Pull request #1353 (2023-04-10 23:24:24)

Tests and app writers check data store type


Patch:
diff --git a/src/cogent3/app/io_new.py b/src/cogent3/app/io_new.py
index da1b4a0db..b655d57b9 100644
--- a/src/cogent3/app/io_new.py
+++ b/src/cogent3/app/io_new.py
@@ -143,9 +143,6 @@ def open_data_store(
     )
     if base_path.is_dir():
         ds_suffix = None
-    elif base_path.suffix == ".tinydb":
-        kwargs["suffix"] = "json"
-        kwargs.pop("mode")
     elif base_path.suffix == ".sqlitedb" or base_path.name == _MEMORY:
         ds_suffix = ".sqlitedb"
         kwargs.pop("suffix")
@@ -507,6 +504,8 @@ class write_json:
             source. The default function removes path information and
             filename + compression suffixes.
         """
+        if not isinstance(data_store, DataStoreABC):
+            raise TypeError(f"invalid type {type(data_store)!r} for data_store")
         self.data_store = data_store
         self._format = "json"
         self._id_from_source = id_from_source
@@ -540,6 +539,8 @@ class write_seqs:
         format
             sequence format
         """
+        if not isinstance(data_store, DataStoreABC):
+            raise TypeError(f"invalid type {type(data_store)!r} for data_store")
         self.data_store = data_store
         self._formatter = FORMATTERS[format]
         self._id_from_source = id_from_source
@@ -572,6 +573,8 @@ class write_tabular:
         format
             tabular format, e.g. 'csv' or 'tsv'
         """
+        if not isinstance(data_store, DataStoreABC):
+            raise TypeError(f"invalid type {type(data_store)!r} for data_store")
         self.data_store = data_store
         self._id_from_source = id_from_source
         self._format = format
@@ -608,6 +611,8 @@ class write_db:
             A callable for serialising input data. By default, it converts
             data into primitive python data types, pickling the result.
         """
+        if not isinstance(data_store, DataStoreABC):
+            raise TypeError(f"invalid type {type(data_store)!r} for data_store")
         self.data_store = data_store
         self._serialiser = serialiser
         self._id_from_source = id_from_source


Test patch:
diff --git a/tests/test_app/test_composable.py b/tests/test_app/test_composable.py
index 30ffbfba4..504d90a9f 100644
--- a/tests/test_app/test_composable.py
+++ b/tests/test_app/test_composable.py
@@ -63,13 +63,12 @@ DATA_DIR = Path(__file__).parent.parent / "data"
 
 
 @pytest.fixture(scope="function")
-def tmp_dir(tmpdir_factory):
-    return tmpdir_factory.mktemp("datastore")
+def tmp_dir(tmp_path_factory):
+    return tmp_path_factory.mktemp("datastore")
 
 
 @pytest.fixture(scope="function")
 def fasta_dir(tmp_dir):
-    tmp_dir = Path(tmp_dir)
     filenames = DATA_DIR.glob("*.fasta")
     fasta_dir = tmp_dir / "fasta"
     fasta_dir.mkdir(parents=True, exist_ok=True)
@@ -81,7 +80,6 @@ def fasta_dir(tmp_dir):
 
 @pytest.fixture(scope="function")
 def write_dir1(tmp_dir):
-    tmp_dir = Path(tmp_dir)
     write_dir1 = tmp_dir / "write1"
     write_dir1.mkdir(parents=True, exist_ok=True)
     yield write_dir1
@@ -90,7 +88,6 @@ def write_dir1(tmp_dir):
 
 @pytest.fixture(scope="function")
 def write_dir2(tmp_dir):
-    tmp_dir = Path(tmp_dir)
     write_dir2 = tmp_dir / "write2"
     write_dir2.mkdir(parents=True, exist_ok=True)
     yield write_dir2
@@ -134,6 +131,15 @@ def full_dstore(write_dir1, nc_objects, completed_objects, log_data):
     return dstore
 
 
+@pytest.fixture(scope="function")
+def nc_dstore(tmp_dir, nc_objects):
+    dstore = DataStoreDirectory(tmp_dir, suffix="fasta", mode=OVERWRITE)
+    for id, data in nc_objects.items():
+        dstore.write_not_completed(unique_id=id, data=data.to_json())
+
+    return dstore
+
+
 @pytest.fixture(scope="function")
 def half_dstore1(write_dir1, nc_objects, completed_objects, log_data):
     dstore = DataStoreDirectory(write_dir1, suffix="fasta", mode=OVERWRITE)
@@ -401,10 +407,24 @@ def test_apply_to_invalid_logger(tmp_dir, logger_val):
     with pytest.raises(TypeError):
         process.apply_to(dstore, show_progress=False, logger=logger_val)
 
-    out_dstore.close()
+
+def test_apply_to_input_only_not_completed(nc_dstore, tmp_dir):
+    """correctly skips notcompleted"""
+    dstore = open_data_store(DATA_DIR, suffix="fasta", limit=3)
+    # trigger creation of notcompleted
+    outpath = tmp_dir / "delme.sqlitedb"
+    out_dstore = open_data_store(outpath, mode="w")
+    writer = io_app.write_db(out_dstore)
+    process = (
+        io_app.load_aligned(format="fasta", moltype="dna")
+        + sample_app.min_length(3000)
+        + writer
+    )
+    process.apply_to(dstore, show_progress=False)
+    assert len(out_dstore.not_completed) == len(nc_dstore)
 
 
-def test_apply_to_not_completed(tmp_dir):
+def test_apply_to_makes_not_completed(tmp_dir):
     """correctly creates notcompleted"""
     dstore = open_data_store(DATA_DIR, suffix="fasta", limit=3)
     reader = io_app.load_aligned(format="fasta", moltype="dna")
diff --git a/tests/test_app/test_dist.py b/tests/test_app/test_dist.py
index 213358211..d7ab5679d 100644
--- a/tests/test_app/test_dist.py
+++ b/tests/test_app/test_dist.py
@@ -1,5 +1,3 @@
-import os
-
 from tempfile import TemporaryDirectory
 from unittest import TestCase, main
 
@@ -46,12 +44,13 @@ _seqs5 = {"Human": "ASSLQHENSSLLLT", "Bandicoot": "XSLMLETSSLLSN"}
 
 
 def _get_all_composable_apps():
+    out_dstore = open_data_store(":memory:", mode="w")
     return [
         get_app("align_to_ref"),
         get_app("progressive_align", model="GY94"),
         get_app("fixed_length", 100),
         get_app("sample.min_length", 100),
-        get_app("write_seqs", os.getcwd()),
+        get_app("write_seqs", out_dstore),
         get_app(
             "omit_bad_seqs",
         ),
diff --git a/tests/test_app/test_io_new.py b/tests/test_app/test_io_new.py
index cc2cd4175..839f3d1a9 100644
--- a/tests/test_app/test_io_new.py
+++ b/tests/test_app/test_io_new.py
@@ -573,6 +573,21 @@ def test_writer_unique_id_arg(tmp_dir, writer, data, dstore):
     assert m.unique_id == expect
 
 
+@pytest.mark.parametrize(
+    "writer",
+    (
+        "write_seqs",
+        "write_db",
+        "write_json",
+        "write_tabular",
+    ),
+)
+def test_writer_fails_on_data_store(writer):
+    # should raise a type error if not a data store provided
+    with pytest.raises(TypeError):
+        get_app(writer, data_store="not-a-data_store")
+
+
 def test_open_suffix_dirname(tmp_dir):
     # open_data_store correctly identifies this as a directory data store
     outpath = tmp_dir / "melsubgroup_aln_flydivas_v1.2"