Repository: freedomofpress/securedrop

Issue:
New submission cleanup could delete valid files of replies
## Description

In `securedrop/management/submissions.py`, the `find_disconnected_fs_submissions` returns any file found in the store that is not associated with a `Submission` record in the database. It should only return files that have neither a `Submission` nor `Reply` record, to avoid warning admins about and possibly deleting replies.

## Steps to Reproduce

- Run `make dev`
- In another shell:
  - Run `docker container ls` to get the dev container's ID
  - Run `docker exec -it "container-id" bash`
  - Run `./manage.py list-disconnected-fs-submissions`

## Expected Behavior

With a consistent database and filesystem, no disconnects should be reported. 

## Actual Behavior

The files of all replies in the database are reported as candidates for cleanup.

## Comments

The function simply needs to build a list of reply files as it does for submissions, and check the files under the store against the complete set of valid files.



Discussion:

Resolving patch:
diff --git a/securedrop/management/submissions.py b/securedrop/management/submissions.py
--- a/securedrop/management/submissions.py
+++ b/securedrop/management/submissions.py
@@ -9,11 +9,14 @@
 
 from db import db
 from rm import secure_delete
-from models import Source, Submission
+from models import Reply, Source, Submission
 from management import app_context
 
 
 def find_disconnected_db_submissions(path):
+    """
+    Finds Submission records whose file does not exist.
+    """
     submissions = db.session.query(Submission).order_by(Submission.id, Submission.filename).all()
 
     files_in_fs = {}
@@ -50,7 +53,7 @@ def list_disconnected_db_submissions(args):
         if disconnected_submissions:
             print(
                 'Run "manage.py delete-disconnected-db-submissions" to delete these records.',
-                file=sys.stderr
+                file=sys.stderr,
             )
         for s in disconnected_submissions:
             print(s.id)
@@ -78,8 +81,14 @@ def delete_disconnected_db_submissions(args):
 
 
 def find_disconnected_fs_submissions(path):
+    """
+    Finds files in the store that lack a Submission or Reply record.
+    """
     submissions = Submission.query.order_by(Submission.id, Submission.filename).all()
-    files_in_db = {s.filename: s for s in submissions}
+    files_in_db = {s.filename: True for s in submissions}
+
+    replies = Reply.query.order_by(Reply.id, Reply.filename).all()
+    files_in_db.update({r.filename: True for r in replies})
 
     files_in_fs = {}
     for directory, subdirs, files in os.walk(path):
@@ -99,7 +108,7 @@ def find_disconnected_fs_submissions(path):
 
 def check_for_disconnected_fs_submissions(args):
     """
-    Check for files without a corresponding Submission record in the database.
+    Check for files without a corresponding Submission or Reply record in the database.
     """
     with app_context():
         disconnected = find_disconnected_fs_submissions(args.store_dir)
@@ -114,14 +123,14 @@ def check_for_disconnected_fs_submissions(args):
 
 def list_disconnected_fs_submissions(args):
     """
-    List files without a corresponding Submission record in the database.
+    List files without a corresponding Submission or Reply record in the database.
     """
     with app_context():
         disconnected_files = find_disconnected_fs_submissions(args.store_dir)
         if disconnected_files:
             print(
                 'Run "manage.py delete-disconnected-fs-submissions" to delete these files.',
-                file=sys.stderr
+                file=sys.stderr,
             )
         for f in disconnected_files:
             print(f)


Test patch:
diff --git a/securedrop/tests/test_submission_cleanup.py b/securedrop/tests/test_submission_cleanup.py
--- a/securedrop/tests/test_submission_cleanup.py
+++ b/securedrop/tests/test_submission_cleanup.py
@@ -13,7 +13,7 @@ def test_delete_disconnected_db_submissions(journalist_app, config):
     Test that Submission records without corresponding files are deleted.
     """
     with journalist_app.app_context():
-        source, codename = utils.db_helper.init_source()
+        source, _ = utils.db_helper.init_source()
         source_id = source.id
 
         # make two submissions
@@ -43,7 +43,7 @@ def test_delete_disconnected_fs_submissions(journalist_app, config):
     """
     Test that files in the store without corresponding Submission records are deleted.
     """
-    source, codename = utils.db_helper.init_source()
+    source, _ = utils.db_helper.init_source()
 
     # make two submissions
     utils.db_helper.submit(source, 2)
@@ -51,6 +51,11 @@ def test_delete_disconnected_fs_submissions(journalist_app, config):
     submission_filename = source.submissions[0].filename
     disconnect_path = os.path.join(config.STORE_DIR, source_filesystem_id, submission_filename)
 
+    # make two replies, to make sure that their files are not seen
+    # as disconnects
+    journalist, _ = utils.db_helper.init_journalist("Mary", "Lane")
+    utils.db_helper.reply(journalist, source, 2)
+
     # delete the first Submission record
     db.session.delete(source.submissions[0])
     db.session.commit()