Repository: cogent3/cogent3
Pull request #1167 (2022-10-10 05:57:31)

DEV: Change _write function and how to add members-Add new test for _…
…limit in DataStoreDirectory


Patch:
diff --git a/src/cogent3/app/data_store_new.py b/src/cogent3/app/data_store_new.py
index a5050fc43..efc1f1b90 100644
--- a/src/cogent3/app/data_store_new.py
+++ b/src/cogent3/app/data_store_new.py
@@ -182,7 +182,8 @@ class DataStoreDirectory(DataStoreABC):
         source = Path(source)
         self.source = source.expanduser()
         self.suffix = suffix
-        self._members = []
+        self._completed = []
+        self._not_completed = []
         self._limit = limit
         self._verbose = verbose
         self._checksums = {}
@@ -253,19 +254,9 @@ class DataStoreDirectory(DataStoreABC):
             file.unlink()
             md5_file = md5_dir / f"{file.stem}.txt"
             md5_file.unlink()
-
-    @property
-    def members(self) -> list[DataMember]:
-        if not self._members:  # members in completed and not_completed
-            self._members = []
-            for path in list(self.source.glob(f"*.{self.suffix}")) + list(
-                (self.source / _NOT_COMPLETED_TABLE).glob("*.json")
-            ):
-                if self._limit and len(self._members) >= self._limit:
-                    break
-                self._members.append(DataMember(self, Path(path).name))
-
-        return self._members
+        Path(self.source / _NOT_COMPLETED_TABLE).rmdir()
+        # reset _not_completed list to force not_completed function to make it again
+        self._not_completed = []
 
     @property
     def logs(self) -> list[DataMember]:
@@ -278,24 +269,31 @@ class DataStoreDirectory(DataStoreABC):
 
     @property
     def completed(self) -> list[DataMember]:
-        path = Path(self.source)
-        return (
-            [DataMember(self, path / m.name) for m in path.glob(f"*.{self.suffix}")]
-            if path.exists()
-            else []
-        )
+        if not self._completed:
+            self._completed = []
+            for i, m in enumerate(self.source.glob(f"*.{self.suffix}")):
+                if self._limit and i == self._limit:
+                    break
+                self._completed.append(DataMember(self, m.name))
+        return self._completed
 
     @property
     def not_completed(self) -> list[DataMember]:
-        nc_dir = self.source / _NOT_COMPLETED_TABLE
-        return (
-            [
-                DataMember(self, Path(_NOT_COMPLETED_TABLE) / m.name)
-                for m in nc_dir.glob("*.json")
-            ]
-            if nc_dir.exists()
-            else []
-        )
+        if not self._not_completed:
+            self._not_completed = []
+            for i, m in enumerate((self.source / _NOT_COMPLETED_TABLE).glob(f"*.json")):
+                if self._limit and i == self._limit:
+                    break
+                self._not_completed.append(
+                    DataMember(self, Path(_NOT_COMPLETED_TABLE) / m.name)
+                )
+        return self._not_completed
+
+    @property
+    def members(self) -> list[DataMember]:
+        return self.completed + self.not_completed
+
+    # need to update write methods to append to _completed and _not_completed
 
     def _write(
         self, subdir: str, unique_id: str, suffix: str, data: str, md5: bool
@@ -305,20 +303,30 @@ class DataStoreDirectory(DataStoreABC):
         # check suffix compatible with this datastore
         sfx, cmp = get_format_suffixes(unique_id)
         if sfx != suffix:
-            unique_id = f"{unique_id}.{suffix}"
+            unique_id = f"{Path(unique_id).stem}.{suffix}"
         unique_id = (
             unique_id.replace(self.suffix, suffix)
             if self.suffix and self.suffix != suffix
             else unique_id
         )
+        if suffix != "log" and unique_id in self:
+            return
+        if not (self.source / subdir).exists():
+            (self.source / subdir).mkdir(parents=True, exist_ok=True)
         with open_(self.source / subdir / unique_id, mode="w") as out:
             out.write(data)
-        if suffix != "log" and not self._limit or len(self) < self._limit:
-            self._members.append(DataMember(self, unique_id))
+        if subdir == _NOT_COMPLETED_TABLE:
+            self._not_completed.append(
+                DataMember(self, Path(_NOT_COMPLETED_TABLE) / unique_id)
+            )
+        elif subdir == "":
+            self._completed.append(DataMember(self, unique_id))
         if md5:
             md5 = get_text_hexdigest(data)
             unique_id = unique_id.replace(suffix, "txt")
             unique_id = unique_id if cmp is None else unique_id.replace(f".{cmp}", "")
+            if not (self.source / _MD5_TABLE).exists():
+                (self.source / _MD5_TABLE).mkdir(parents=True, exist_ok=True)
             with open_(self.source / _MD5_TABLE / unique_id, mode="w") as out:
                 out.write(md5)
 
@@ -415,11 +423,10 @@ class DataStoreDirectory(DataStoreABC):
         return Table(header=header, data=rows, title="not completed records")
 
     def __repr__(self):
-        num_completed = len(self)
-        num_not_completed = len(self.not_completed)
+        num = len(self.members)
         name = self.__class__.__name__
-        sample = f"{list(self[:2])}..." if num_completed > 2 else list(self)
-        return f"{num_completed + num_not_completed}x member {name}(source='{self.source}', members={sample})"
+        sample = f"{list(self[:2])}..." if num > 2 else list(self)
+        return f"{num}x member {name}(source='{self.source}', members={sample})"
 
 
 @singledispatch


Test patch:
diff --git a/tests/test_app/test_data_store_new.py b/tests/test_app/test_data_store_new.py
index bce3bc7d5..c467067c6 100644
--- a/tests/test_app/test_data_store_new.py
+++ b/tests/test_app/test_data_store_new.py
@@ -54,25 +54,20 @@ def fasta_dir(tmp_dir):
     return fasta_dir
 
 
-@pytest.fixture(scope="session")
+@pytest.fixture(scope="function")
 def write_dir(tmp_dir):
     tmp_dir = Path(tmp_dir)
     write_dir = tmp_dir / "write"
     write_dir.mkdir(parents=True, exist_ok=True)
-    return write_dir
+    yield write_dir
+    shutil.rmtree(write_dir)
 
 
 @pytest.fixture(scope="function")
 def nc_dir(tmp_dir):
     tmp_dir = Path(tmp_dir)
     nc_dir = tmp_dir / "nc_dir"
-    logs_dir = nc_dir / _LOG_TABLE
-    not_dir = nc_dir / _NOT_COMPLETED_TABLE
-    md5_dir = nc_dir / _MD5_TABLE
     nc_dir.mkdir(parents=True, exist_ok=True)
-    logs_dir.mkdir(exist_ok=True)
-    not_dir.mkdir(exist_ok=True)
-    (logs_dir / "scitrack.log").write_text((DATA_DIR / "scitrack.log").read_text())
     yield nc_dir
     shutil.rmtree(nc_dir)
 
@@ -82,7 +77,7 @@ def ro_dstore(fasta_dir):
     return DataStoreDirectory(fasta_dir, suffix="fasta", if_dest_exists=READONLY)
 
 
-@pytest.fixture(scope="session")
+@pytest.fixture(scope="function")
 def w_dstore(write_dir):
     return DataStoreDirectory(write_dir, suffix="fasta", if_dest_exists=OVERWRITE)
 
@@ -90,6 +85,10 @@ def w_dstore(write_dir):
 @pytest.fixture(scope="function")
 def nc_dstore(nc_dir):
     dstore = DataStoreDirectory(nc_dir, suffix="fasta", if_dest_exists=SKIP)
+    # write one log file
+    log_filename = "scitrack.log"
+    dstore.write_log(log_filename, (DATA_DIR / log_filename).read_text())
+    # write three not_completed file
     nc = [
         NotCompleted(
             "FAIL", f"dummy{i}", f"dummy_message{i}", source=f"dummy_source{i}"
@@ -101,6 +100,7 @@ def nc_dstore(nc_dir):
     assert len(dstore.not_completed) == 3
     assert len(list((nc_dir / _MD5_TABLE).glob("*.txt"))) == len(dstore)
     filenames = DATA_DIR.glob("*.fasta")
+    # write six fasta file
     for fn in filenames:
         identifier = fn.name
         dstore.write(identifier, fn.read_text())
@@ -235,13 +235,17 @@ def test_notcompleted(nc_dstore):
 
 
 def test_no_not_completed_subdir(nc_dstore):
+    expect = f"{len(nc_dstore.completed)+len(nc_dstore.not_completed)}x member"
+    assert repr(nc_dstore).startswith(expect)
     # first remove not_completed directory
     nc_dstore.drop_not_completed()
-    Path(nc_dstore.source / _NOT_COMPLETED_TABLE).rmdir()
     # test repr work without not_completed directory
     assert not Path(nc_dstore.source / _NOT_COMPLETED_TABLE).exists()
-    expect = "9x member"
+    expect = f"{len(nc_dstore.completed)}x member"
+    assert repr(nc_dstore).startswith(expect)
+    expect = f"{len(nc_dstore)}x member"
     assert repr(nc_dstore).startswith(expect)
+    assert len(nc_dstore) == len(nc_dstore.completed)
     not_dir = nc_dstore.source / _NOT_COMPLETED_TABLE
     not_dir.mkdir(exist_ok=True)
 
@@ -258,3 +262,11 @@ def test_drop_not_completed(nc_dstore):
     assert len(nc_dstore.not_completed) == 0
     num_md5 = len(list((nc_dstore.source / _MD5_TABLE).glob("*.txt")))
     assert num_md5 == num_completed
+
+
+def test_limit_datastore(nc_dstore):
+    assert len(nc_dstore) == len(nc_dstore.completed) + len(nc_dstore.not_completed)
+    nc_dstore._limit = len(nc_dstore.completed)
+    nc_dstore.drop_not_completed()
+    assert len(nc_dstore) == nc_dstore._limit
+    assert len(nc_dstore) == len(nc_dstore.completed)