Repository: virtool/virtool
Issue: Clean up orphaned analysis directories

At some point during Virtool's development history, analyses deleted from the database, were not cleanly removed from the filesystem.

Create a background process for cleaning up the orphaned directories.

It may be a good idea to make a reusable framework for similar background activities. Consider moving components of the `virtool.organize` module to this system.

Changed files:
virtool/organize.py, virtool/db/analyses.py, virtool/app.py

diff --git a/virtool/app.py b/virtool/app.py
index d436743da..00f294218 100644
--- a/virtool/app.py
+++ b/virtool/app.py
@@ -177,7 +177,7 @@ async def init_check_db(app):
     db = app["db"]
 
     logger.info("Checking database...")
-    await virtool.organize.organize(db, app["settings"], app["version"])
+    await virtool.organize.organize(app)
 
     logger.info("Creating database indexes...")
     await db.analyses.create_index("sample.id")
diff --git a/virtool/db/analyses.py b/virtool/db/analyses.py
index 6734c4d75..b3a7c014a 100644
--- a/virtool/db/analyses.py
+++ b/virtool/db/analyses.py
@@ -1,5 +1,6 @@
 import asyncio
 import json
+import os
 
 import aiofiles
 
@@ -262,3 +263,25 @@ async def update_nuvs_blast(db, settings, analysis_id, sequence_index, rid):
     })
 
     return data, document
+
+
+async def remove_orphaned_directories(app):
+    """
+    Remove all analysis directories for which an analysis document does not exist in the database.
+
+    :param app:
+    """
+    db = app["db"]
+
+    samples_path = os.path.join(app["settings"]["data_path"], "samples")
+
+    existing_ids = set(await db.analyses.distinct("_id"))
+
+    for sample_id in os.listdir(samples_path):
+        analyses_path = os.path.join(samples_path, sample_id, "analysis")
+
+        to_delete = set(os.listdir(analyses_path)) - existing_ids
+
+        for analysis_id in to_delete:
+            analysis_path = os.path.join(analyses_path, analysis_id)
+            await app["run_in_thread"](virtool.utils.rm, analysis_path, True)
diff --git a/virtool/organize.py b/virtool/organize.py
index 772e094e3..a7f01491b 100644
--- a/virtool/organize.py
+++ b/virtool/organize.py
@@ -2,6 +2,7 @@ import logging
 
 import pymongo.errors
 
+import virtool.db.analyses
 import virtool.db.history
 import virtool.db.otus
 import virtool.db.references
@@ -19,7 +20,12 @@ async def delete_unready(collection):
     await collection.delete_many({"ready": False})
 
 
-async def organize(db, settings, server_version):
+async def organize(app):
+    db = app["db"]
+    settings = app["settings"]
+    server_version = app["version"]
+
+    await organize_analyses(app)
     await organize_files(db)
     await organize_groups(db)
     await organize_status(db, server_version)
@@ -27,6 +33,17 @@ async def organize(db, settings, server_version):
     await organize_samples(db)
 
 
+async def organize_analyses(app):
+    """
+    Remove orphaned analysis directories.
+
+    :param app:
+    """
+    logger.info(" • analyses")
+
+    await virtool.db.analyses.remove_orphaned_directories(app)
+
+
 async def organize_files(db):
     """
     Make all files unreserved. This is only called when the server first starts.