Repository: freedomofpress/securedrop

Issue:
Handle Submissions with no Sources in currently running instances.
We need to handle submissions database entries in currently running instances that have no Source. This is a follow up issue to https://github.com/freedomofpress/securedrop/issues/1188.



Discussion:
@garrettr Can you add this as a milestone? Thanks!

X-link: https://github.com/freedomofpress/securedrop/issues/1419.
note: the correct place to address this ticket now is in an alembic revision (with no `downgrade` implemented as this is a destructive action)

Resolving patch:
diff --git a/securedrop/alembic/versions/3da3fcab826a_delete_orphaned_submissions.py b/securedrop/alembic/versions/3da3fcab826a_delete_orphaned_submissions.py
new file mode 100644
--- /dev/null
+++ b/securedrop/alembic/versions/3da3fcab826a_delete_orphaned_submissions.py
@@ -0,0 +1,107 @@
+"""delete orphaned submissions and replies
+
+Ref: https://github.com/freedomofpress/securedrop/issues/1189
+
+Revision ID: 3da3fcab826a
+Revises: 60f41bb14d98
+Create Date: 2018-11-25 19:40:25.873292
+
+"""
+import os
+from alembic import op
+import sqlalchemy as sa
+from rm import secure_delete
+
+# raise the errors if we're not in production
+raise_errors = os.environ.get("SECUREDROP_ENV", "prod") != "prod"
+
+try:
+    from journalist_app import create_app
+    from sdconfig import config
+    from store import NoFileFoundException, TooManyFilesException
+    from worker import create_queue
+except ImportError:
+    # This is a fresh install, and config.py has not been created yet.
+    if raise_errors:
+        raise
+
+# revision identifiers, used by Alembic.
+revision = '3da3fcab826a'
+down_revision = '60f41bb14d98'
+branch_labels = None
+depends_on = None
+
+
+def raw_sql_grab_orphaned_objects(table_name):
+    """Objects that have a source ID that doesn't exist in the
+    sources table OR a NULL source ID should be deleted."""
+    return ('SELECT id, filename, source_id FROM {table} '  # nosec
+            'WHERE source_id NOT IN (SELECT id FROM sources) '
+            'UNION SELECT id, filename, source_id FROM {table} '  # nosec
+            'WHERE source_id IS NULL').format(table=table_name)
+
+
+def upgrade():
+    conn = op.get_bind()
+    submissions = conn.execute(
+        sa.text(raw_sql_grab_orphaned_objects('submissions'))
+    ).fetchall()
+
+    replies = conn.execute(
+        sa.text(raw_sql_grab_orphaned_objects('replies'))
+    ).fetchall()
+
+    try:
+        app = create_app(config)
+        with app.app_context():
+            for submission in submissions:
+                try:
+                    conn.execute(
+                    sa.text("""
+                        DELETE FROM submissions
+                        WHERE id=:id
+                    """).bindparams(id=submission.id)
+                    )
+
+                    file_path = app.storage.path_without_filesystem_id(submission.filename)
+                    create_queue().enqueue(secure_delete, file_path)
+                except NoFileFoundException:
+                    # The file must have been deleted by the admin, remove the row
+                    conn.execute(
+                    sa.text("""
+                        DELETE FROM submissions
+                        WHERE id=:id
+                    """).bindparams(id=submission.id)
+                    )
+                except TooManyFilesException:
+                    pass
+
+            for reply in replies:
+                try:
+                    conn.execute(
+                        sa.text("""
+                            DELETE FROM replies
+                            WHERE id=:id
+                        """).bindparams(id=reply.id)
+                    )
+
+                    file_path = app.storage.path_without_filesystem_id(reply.filename)
+                    create_queue().enqueue(secure_delete, file_path)
+                except NoFileFoundException:
+                    # The file must have been deleted by the admin, remove the row
+                    conn.execute(
+                        sa.text("""
+                            DELETE FROM replies
+                            WHERE id=:id
+                        """).bindparams(id=reply.id)
+                    )
+                except TooManyFilesException:
+                    pass
+    except:  # noqa
+        if raise_errors:
+            raise
+
+
+def downgrade():
+    # This is a destructive alembic migration, it cannot be downgraded
+    pass
diff --git a/securedrop/store.py b/securedrop/store.py
--- a/securedrop/store.py
+++ b/securedrop/store.py
@@ -43,6 +43,23 @@ class PathException(Exception):
     pass
 
 
+class TooManyFilesException(Exception):
+    """An exception raised by path_without_filesystem_id when too many
+    files has been found for a given submission or reply.
+    This could be due to a very unlikely collision between
+    journalist_designations.
+    """
+    pass
+
+
+class NoFileFoundException(Exception):
+    """An exception raised by path_without_filesystem_id when a file could
+    not be found for a given submission or reply.
+    This is likely due to an admin manually deleting files from the server.
+    """
+    pass
+
+
 class NotEncrypted(Exception):
     """An exception raised if a file expected to be encrypted client-side
     is actually plaintext.
@@ -107,6 +124,29 @@ def path(self, *s):
         self.verify(absolute)
         return absolute
 
+    def path_without_filesystem_id(self, filename):
+        # type: (str) -> str
+        """Get the normalized, absolute file path, within
+           `self.__storage_path` for a filename when the filesystem_id
+           is not known.
+        """
+
+        joined_paths = []
+        for rootdir, _, files in os.walk(os.path.abspath(self.__storage_path)):
+            for file_ in files:
+                if file_ in filename:
+                    joined_paths.append(os.path.join(rootdir, file_))
+
+        if len(joined_paths) > 1:
+            raise TooManyFilesException('Found duplicate files!')
+        elif len(joined_paths) == 0:
+            raise NoFileFoundException('File not found: {}'.format(filename))
+        else:
+            absolute = joined_paths[0]
+
+        self.verify(absolute)
+        return absolute
+
     def get_bulk_archive(self, selected_submissions, zip_directory=''):
         # type: (List, str) -> _TemporaryFileWrapper
         """Generate a zip file from the selected submissions"""


Test patch:
diff --git a/securedrop/tests/migrations/helpers.py b/securedrop/tests/migrations/helpers.py
--- a/securedrop/tests/migrations/helpers.py
+++ b/securedrop/tests/migrations/helpers.py
@@ -36,6 +36,10 @@ def random_chars(len, chars=string.printable):
     return ''.join([random.choice(chars) for _ in range(len)])
 
 
+def random_ascii_chars(len, chars=string.ascii_lowercase):
+    return ''.join([random.choice(chars) for _ in range(len)])
+
+
 def random_datetime(nullable):
     if nullable and random_bool():
         return None
diff --git a/securedrop/tests/migrations/migration_3da3fcab826a.py b/securedrop/tests/migrations/migration_3da3fcab826a.py
new file mode 100644
--- /dev/null
+++ b/securedrop/tests/migrations/migration_3da3fcab826a.py
@@ -0,0 +1,161 @@
+# -*- coding: utf-8 -*-
+
+import random
+import os
+from uuid import uuid4
+
+from sqlalchemy import text
+
+from db import db
+from journalist_app import create_app
+from .helpers import random_bool, random_chars, random_ascii_chars, random_datetime, bool_or_none
+
+
+TEST_DATA_DIR = '/tmp/securedrop/store'
+
+
+def create_file_in_dummy_source_dir(filename):
+    filesystem_id = 'dummy'
+    basedir = os.path.join(TEST_DATA_DIR, filesystem_id)
+
+    if not os.path.exists(basedir):
+        os.makedirs(basedir)
+
+    path_to_file = os.path.join(basedir, filename)
+    with open(path_to_file, 'a'):
+        os.utime(path_to_file, None)
+
+
+class UpgradeTester:
+
+    """This migration verifies that any orphaned submission or reply data from
+    deleted sources is also deleted.
+    """
+
+    def __init__(self, config):
+        self.config = config
+        self.app = create_app(config)
+        self.journalist_id = None
+
+    def load_data(self):
+        with self.app.app_context():
+            self.create_journalist()
+            self.add_source()
+            self.valid_source_id = 1
+            deleted_source_id = 2
+
+            # Add submissions and replies with and without a valid source
+            self.add_submission(self.valid_source_id)
+            self.add_submission(deleted_source_id)
+            self.add_submission(deleted_source_id, with_file=False)
+            self.add_submission(None)  # NULL source
+
+            self.add_reply(self.journalist_id, self.valid_source_id)
+            self.add_reply(self.journalist_id, deleted_source_id)
+            self.add_reply(self.journalist_id, deleted_source_id, with_file=False)
+            self.add_reply(self.journalist_id, None)  # NULL source
+
+            db.session.commit()
+
+    def create_journalist(self):
+        if self.journalist_id is not None:
+            raise RuntimeError('Journalist already created')
+
+        params = {
+            'uuid': str(uuid4()),
+            'username': random_chars(50),
+            'session_nonce': 0
+        }
+        sql = '''INSERT INTO journalists (uuid, username, session_nonce)
+                 VALUES (:uuid, :username, :session_nonce)
+              '''
+        self.journalist_id = db.engine.execute(text(sql), **params).lastrowid
+
+    def add_reply(self, journalist_id, source_id, with_file=True):
+        filename = '1-' + random_ascii_chars(5) + '-' + random_ascii_chars(5) + '-reply.gpg'
+        params = {
+            'uuid': str(uuid4()),
+            'journalist_id': journalist_id,
+            'source_id': source_id,
+            'filename': filename,
+            'size': random.randint(0, 1024 * 1024 * 500),
+            'deleted_by_source': False,
+        }
+        sql = '''INSERT INTO replies (journalist_id, uuid, source_id, filename,
+                    size, deleted_by_source)
+                 VALUES (:journalist_id, :uuid, :source_id, :filename, :size,
+                         :deleted_by_source)
+              '''
+        db.engine.execute(text(sql), **params)
+
+        if with_file:
+            create_file_in_dummy_source_dir(filename)
+
+    @staticmethod
+    def add_source():
+        filesystem_id = random_chars(96) if random_bool() else None
+        params = {
+            'uuid': str(uuid4()),
+            'filesystem_id': filesystem_id,
+            'journalist_designation': random_chars(50),
+            'flagged': bool_or_none(),
+            'last_updated': random_datetime(nullable=True),
+            'pending': bool_or_none(),
+            'interaction_count': random.randint(0, 1000),
+        }
+        sql = '''INSERT INTO sources (uuid, filesystem_id,
+                    journalist_designation, flagged, last_updated, pending,
+                    interaction_count)
+                 VALUES (:uuid, :filesystem_id, :journalist_designation,
+                    :flagged, :last_updated, :pending, :interaction_count)
+              '''
+        db.engine.execute(text(sql), **params)
+
+    def add_submission(self, source_id, with_file=True):
+        filename = '1-' + random_ascii_chars(5) + '-' + random_ascii_chars(5) + '-doc.gz.gpg'
+        params = {
+            'uuid': str(uuid4()),
+            'source_id': source_id,
+            'filename': filename,
+            'size': random.randint(0, 1024 * 1024 * 500),
+            'downloaded': bool_or_none(),
+        }
+        sql = '''INSERT INTO submissions (uuid, source_id, filename, size,
+                    downloaded)
+                 VALUES (:uuid, :source_id, :filename, :size, :downloaded)
+              '''
+        db.engine.execute(text(sql), **params)
+
+        if with_file:
+            create_file_in_dummy_source_dir(filename)
+
+    def check_upgrade(self):
+        with self.app.app_context():
+            submissions = db.engine.execute(
+                text('SELECT * FROM submissions')).fetchall()
+
+            # Submissions without a source should be deleted
+            assert len(submissions) == 1
+            for submission in submissions:
+                assert submission.source_id == self.valid_source_id
+
+            replies = db.engine.execute(
+                text('SELECT * FROM replies')).fetchall()
+
+            # Replies without a source should be deleted
+            assert len(replies) == 1
+            for reply in replies:
+                assert reply.source_id == self.valid_source_id
+
+
+class DowngradeTester:
+    # This is a destructive alembic migration, it cannot be downgraded
+
+    def __init__(self, config):
+        self.config = config
+
+    def load_data(self):
+        pass
+
+    def check_downgrade(self):
+        pass
diff --git a/securedrop/tests/test_store.py b/securedrop/tests/test_store.py
--- a/securedrop/tests/test_store.py
+++ b/securedrop/tests/test_store.py
@@ -51,6 +51,49 @@ def test_path_returns_filename_of_items_within_folder(journalist_app, config):
     assert generated_absolute_path == expected_absolute_path
 
 
+def test_path_without_filesystem_id(journalist_app, config):
+    filesystem_id = 'example'
+    item_filename = '1-quintuple_cant-msg.gpg'
+
+    basedir = os.path.join(config.STORE_DIR, filesystem_id)
+    os.makedirs(basedir)
+
+    path_to_file = os.path.join(basedir, item_filename)
+    with open(path_to_file, 'a'):
+        os.utime(path_to_file, None)
+
+    generated_absolute_path = \
+        journalist_app.storage.path_without_filesystem_id(item_filename)
+
+    expected_absolute_path = os.path.join(config.STORE_DIR,
+                                          filesystem_id, item_filename)
+    assert generated_absolute_path == expected_absolute_path
+
+
+def test_path_without_filesystem_id_duplicate_files(journalist_app, config):
+    filesystem_id = 'example'
+    filesystem_id_duplicate = 'example2'
+    item_filename = '1-quintuple_cant-msg.gpg'
+
+    basedir = os.path.join(config.STORE_DIR, filesystem_id)
+    duplicate_basedir = os.path.join(config.STORE_DIR, filesystem_id_duplicate)
+
+    for directory in [basedir, duplicate_basedir]:
+        os.makedirs(directory)
+        path_to_file = os.path.join(directory, item_filename)
+        with open(path_to_file, 'a'):
+            os.utime(path_to_file, None)
+
+    with pytest.raises(store.TooManyFilesException):
+        journalist_app.storage.path_without_filesystem_id(item_filename)
+
+
+def test_path_without_filesystem_id_no_file(journalist_app, config):
+    item_filename = 'not there'
+    with pytest.raises(store.NoFileFoundException):
+        journalist_app.storage.path_without_filesystem_id(item_filename)
+
+
 def test_verify_path_not_absolute(journalist_app, config):
     with pytest.raises(store.PathException):
         journalist_app.storage.verify(