Repository: cogent3/cogent3
Pull request #1273 (2023-01-17 04:22:36)

ENH: log db conversions


Patch:
diff --git a/src/cogent3/app/composable.py b/src/cogent3/app/composable.py
index a07ba32fa..4c7cfa820 100644
--- a/src/cogent3/app/composable.py
+++ b/src/cogent3/app/composable.py
@@ -52,9 +52,12 @@ _builtin_seqs = list, set, tuple
 
 
 def _make_logfile_name(process):
-    text = str(process)
-    text = re.split(r"\s+\+\s+", text)
-    parts = [part[: part.find("(")] for part in text]
+    text = re.split(r"\s+\+\s+", str(process))
+    parts = []
+    for part in text:
+        if index := part.find("(") >= 0:
+            part = part[:index]
+        parts.append(part)
     result = "-".join(parts)
     uid = str(uuid4())
     return f"{result}-{uid[:8]}.log"
diff --git a/src/cogent3/app/data_store_new.py b/src/cogent3/app/data_store_new.py
index 6a0a32c50..e5b29f1d1 100644
--- a/src/cogent3/app/data_store_new.py
+++ b/src/cogent3/app/data_store_new.py
@@ -1,5 +1,6 @@
 from __future__ import annotations
 
+import contextlib
 import inspect
 import json
 import re
@@ -589,22 +590,23 @@ def convert_directory_datastore(
 
 
 def convert_tinydb_to_sqlite(source: Path, dest: Optional[Path] = None) -> DataStoreABC:
-    try:
-        from fnmatch import translate
+    from datetime import datetime
+    from fnmatch import translate
+
+    from .composable import CachingLogger, _make_logfile_name
+    from .data_store import load_record_from_json
+    from .io_new import write_db
+    from .sqlite_data_store import _LOG_TABLE, DataStoreSqlite
 
+    try:
         from tinydb import Query, TinyDB
         from tinydb.middlewares import CachingMiddleware
         from tinydb.storages import JSONStorage
-
-        from cogent3.app.data_store import load_record_from_json
-        from cogent3.app.sqlite_data_store import DataStoreSqlite
     except ImportError as e:
         raise ImportError(
             "You need to install tinydb to be able to migrate to new datastore."
         ) from e
 
-    from .io_new import write_db
-
     source = Path(source)
     storage = CachingMiddleware(JSONStorage)
     tinydb = TinyDB(str(source), storage=storage)
@@ -626,14 +628,37 @@ def convert_tinydb_to_sqlite(source: Path, dest: Optional[Path] = None) -> DataS
         raise IOError(
             f"Destination file {str(dest)} already exists. Delete or define new dest."
         )
+
+    LOGGER = CachingLogger(create_dir=True)
+    log_file_path = source.parent / _make_logfile_name("convert_tinydb_to_sqlite")
+    LOGGER.log_file_path = log_file_path
+
     dstore = DataStoreSqlite(source=dest, mode=OVERWRITE)
     writer = write_db(data_store=dstore)
     for id, data, is_completed in data_list:
         if id.endswith(".log"):
-            writer.data_store.write_log(unique_id=id, data=data)
+            cmnd = f"UPDATE {_LOG_TABLE} SET data =?, log_name =?"
+            values = (data, id)
+            with contextlib.suppress(ValueError):
+                date = datetime.strptime(
+                    data.split("\t", maxsplit=1)[0], "%Y-%m-%d %H:%M:%S"
+                )
+                cmnd = f"{cmnd}, date=?"
+                values += (date,)
+
+            cmnd = f"{cmnd} WHERE log_id=?"
+            values += (dstore._log_id,)
+
+            dstore.db.execute(cmnd, values)
         else:
             writer.main(data, identifier=id)
 
+    # add a new log, recording this conversion
+    LOGGER.shutdown()
+    dstore.close()
+    dstore = DataStoreSqlite(source=dest, mode=APPEND)
+    dstore.write_log(unique_id=log_file_path.name, data=log_file_path.read_text())
+    log_file_path.unlink()
     if lock_id is not None or dstore._lock_id:
         cmnd = f"UPDATE state SET lock_pid =? WHERE state_id == 1"
         dstore.db.execute(cmnd, (lock_id,))
diff --git a/src/cogent3/app/sqlite_data_store.py b/src/cogent3/app/sqlite_data_store.py
index 940360b8e..ef5947367 100644
--- a/src/cogent3/app/sqlite_data_store.py
+++ b/src/cogent3/app/sqlite_data_store.py
@@ -256,6 +256,7 @@ class DataStoreSqlite(DataStoreABC):
             self._init_log()
 
         if table_name == _LOG_TABLE:
+            # todo how to evaluate whether writing a new log?
             cmnd = f"UPDATE {table_name} SET data =?, log_name =? WHERE log_id=?"
             self.db.execute(cmnd, (data, unique_id, self._log_id))
             return None


Test patch:
diff --git a/doc/data/demo-locked.tinydb b/tests/data/sample_locked_w_log.tinydb
similarity index 100%
rename from doc/data/demo-locked.tinydb
rename to tests/data/sample_locked_w_log.tinydb
diff --git a/tests/test_app/test_data_store_new.py b/tests/test_app/test_data_store_new.py
index e12d70c9a..911ffa6dc 100644
--- a/tests/test_app/test_data_store_new.py
+++ b/tests/test_app/test_data_store_new.py
@@ -245,8 +245,8 @@ def test_convert_tinydb_to_sqlite_error(tmp_dir):
 @pytest.mark.parametrize(
     "orig,num_logs",
     (
-        (DATA_DIR / "sample_locked.tinydb", 0),
-        (DATA_DIR.parent.parent / "doc" / "data" / "demo-locked.tinydb", 1),
+        (DATA_DIR / "sample_locked.tinydb", 1),
+        (DATA_DIR / "sample_locked_w_log.tinydb", 2),
     ),
 )
 def test_convert_tinydbs_to_sqlite(tmp_dir, orig, num_logs):
diff --git a/tests/test_app/test_sqlite_data_store.py b/tests/test_app/test_sqlite_data_store.py
index 2ceebba1e..eaaa2bf99 100644
--- a/tests/test_app/test_sqlite_data_store.py
+++ b/tests/test_app/test_sqlite_data_store.py
@@ -557,7 +557,7 @@ def _make_appendable_dstore(path, suffix):
 def _make_and_run_proc(out_path, suffix, members):
     out_dstore = _make_appendable_dstore(out_path, suffix)
     loader = get_app("load_unaligned", moltype="dna", format="fasta")
-    mlength = get_app("min_length", 400)
+    mlength = get_app("sample.min_length", 400)
 
     if suffix:
         writer = get_app("write_seqs", out_dstore, format="fasta")