Repository: freedomofpress/securedrop

Issue:
[reply refactor] Preserve conversation history between sources and journalists
In order for journalists to view the conversation history between sources and journalists, when a source deletes the journalist reply from the _source inbox_, it should not delete the reply from the _journalist's_ inbox. This is a major UX win for journalists while preserving the current behavior of minimizing data available if a source codename is compromised. 

We could do this by adding a column to the replies table that `allow_source_read` (`default=True`) or something similar that we can filter by when showing replies to sources. When the source deletes the reply from their inbox, we set this to `False`. 

Epic: #3097


Discussion:
In #3097 it was noted that the message in the source interface ("[Deleting replies] also lets us know that you are aware of our reply") is inaccurate. If we have the `allow_source_read` column, we could potentially also use this to indicate to the journalist that a reply has been read and acknowledged by the source.

At the risk of complicating things, I think there's a case to do so, since a savvy journalist who paid attention to the deletion status of replies to infer source activity would no longer be able to do so after this change.

I suggest we add a follow-up issue for that, where we could discuss the best way to address the UX (both for the web interface and the graphical client). If we don't get to that before 0.9.0, we can remove the inaccurate sentence from the source interface for now.

Resolving patch:
diff --git a/securedrop/alembic/versions/e0a525cbab83_add_column_to_track_source_deletion_of_.py b/securedrop/alembic/versions/e0a525cbab83_add_column_to_track_source_deletion_of_.py
new file mode 100644
--- /dev/null
+++ b/securedrop/alembic/versions/e0a525cbab83_add_column_to_track_source_deletion_of_.py
@@ -0,0 +1,66 @@
+"""add column to track source deletion of replies
+
+Revision ID: e0a525cbab83
+Revises: 2d0ce3ee5bdc
+Create Date: 2018-08-02 00:07:59.242510
+
+"""
+from alembic import op
+import sqlalchemy as sa
+
+
+# revision identifiers, used by Alembic.
+revision = 'e0a525cbab83'
+down_revision = '2d0ce3ee5bdc'
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    # Schema migration
+    op.rename_table('replies', 'replies_tmp')
+
+    # Add new column.
+    op.add_column('replies_tmp',
+                  sa.Column('deleted_by_source', sa.Boolean()))
+
+    # Populate deleted_by_source column in replies_tmp table.
+    conn = op.get_bind()
+    replies = conn.execute(
+        sa.text("SELECT * FROM replies_tmp")).fetchall()
+
+    for reply in replies:
+        id = reply.id
+        conn.execute(
+            sa.text("""UPDATE replies_tmp
+                       SET deleted_by_source=('0')
+                       WHERE id={}""".format(id)))
+
+    # Now create new table with not null constraint applied to
+    # deleted_by_source.
+    op.create_table('replies',
+        sa.Column('id', sa.Integer(), nullable=False),
+        sa.Column('journalist_id', sa.Integer(), nullable=True),
+        sa.Column('source_id', sa.Integer(), nullable=True),
+        sa.Column('filename', sa.String(length=255), nullable=False),
+        sa.Column('size', sa.Integer(), nullable=False),
+        sa.Column('deleted_by_source', sa.Boolean(), nullable=False),
+        sa.ForeignKeyConstraint(['journalist_id'], ['journalists.id'], ),
+        sa.ForeignKeyConstraint(['source_id'], ['sources.id'], ),
+        sa.PrimaryKeyConstraint('id')
+    )
+
+    # Data Migration: move all replies into the new table.
+    conn.execute('''
+        INSERT INTO replies
+        SELECT id, journalist_id, source_id, filename, size, deleted_by_source
+        FROM replies_tmp
+    ''')
+
+    # Now delete the old table.
+    op.drop_table('replies_tmp')
+
+
+def downgrade():
+    with op.batch_alter_table('replies', schema=None) as batch_op:
+        batch_op.drop_column('deleted_by_source')
diff --git a/securedrop/models.py b/securedrop/models.py
--- a/securedrop/models.py
+++ b/securedrop/models.py
@@ -217,6 +217,8 @@ class Reply(db.Model):
     filename = Column(String(255), nullable=False)
     size = Column(Integer, nullable=False)
 
+    deleted_by_source = Column(Boolean, default=False, nullable=False)
+
     def __init__(self, journalist, source, filename):
         self.journalist_id = journalist.id
         self.source_id = source.id
diff --git a/securedrop/source_app/main.py b/securedrop/source_app/main.py
--- a/securedrop/source_app/main.py
+++ b/securedrop/source_app/main.py
@@ -10,7 +10,6 @@
 
 from db import db
 from models import Source, Submission, Reply, get_one_or_else
-from rm import srm
 from source_app.decorators import login_required
 from source_app.utils import (logged_in, generate_unique_codename,
                               async_genkey, normalize_timestamps,
@@ -76,7 +75,10 @@ def create():
     @login_required
     def lookup():
         replies = []
-        for reply in g.source.replies:
+        source_inbox = Reply.query.filter(Reply.source_id == g.source.id) \
+                                  .filter(Reply.deleted_by_source == False).all()  # noqa
+
+        for reply in source_inbox:
             reply_path = current_app.storage.path(
                 g.filesystem_id,
                 reply.filename,
@@ -203,11 +205,16 @@ def submit():
     @view.route('/delete', methods=('POST',))
     @login_required
     def delete():
+        """This deletes the reply from the source's inbox, but preserves
+        the history for journalists such that they can view conversation
+        history.
+        """
+
         query = Reply.query.filter(
             Reply.filename == request.form['reply_filename'])
         reply = get_one_or_else(query, current_app.logger, abort)
-        srm(current_app.storage.path(g.filesystem_id, reply.filename))
-        db.session.delete(reply)
+        reply.deleted_by_source = True
+        db.session.add(reply)
         db.session.commit()
 
         flash(gettext("Reply deleted"), "notification")
@@ -216,15 +223,16 @@ def delete():
     @view.route('/delete-all', methods=('POST',))
     @login_required
     def batch_delete():
-        replies = g.source.replies
+        replies = Reply.query.filter(Reply.source_id == g.source.id) \
+                             .filter(Reply.deleted_by_source == False).all()  # noqa
         if len(replies) == 0:
             current_app.logger.error("Found no replies when at least one was "
                                      "expected")
             return redirect(url_for('.lookup'))
 
         for reply in replies:
-            srm(current_app.storage.path(g.filesystem_id, reply.filename))
-            db.session.delete(reply)
+            reply.deleted_by_source = True
+            db.session.add(reply)
         db.session.commit()
 
         flash(gettext("All replies have been deleted"), "notification")


Test patch:
diff --git a/securedrop/tests/migrations/migration_e0a525cbab83.py b/securedrop/tests/migrations/migration_e0a525cbab83.py
new file mode 100644
--- /dev/null
+++ b/securedrop/tests/migrations/migration_e0a525cbab83.py
@@ -0,0 +1,177 @@
+# -*- coding: utf-8 -*-
+
+import random
+import string
+import uuid
+
+from sqlalchemy import text
+from sqlalchemy.exc import NoSuchColumnError
+
+from db import db
+from journalist_app import create_app
+from .helpers import (random_bool, random_bytes, random_chars, random_datetime,
+                      random_username, bool_or_none)
+
+random.seed('ᕕ( ᐛ )ᕗ')
+
+
+def add_source():
+    filesystem_id = random_chars(96) if random_bool() else None
+    params = {
+        'uuid': str(uuid.uuid4()),
+        'filesystem_id': filesystem_id,
+        'journalist_designation': random_chars(50),
+        'flagged': bool_or_none(),
+        'last_updated': random_datetime(nullable=True),
+        'pending': bool_or_none(),
+        'interaction_count': random.randint(0, 1000),
+    }
+    sql = '''INSERT INTO sources (uuid, filesystem_id,
+                journalist_designation, flagged, last_updated, pending,
+                interaction_count)
+             VALUES (:uuid, :filesystem_id, :journalist_designation,
+                :flagged, :last_updated, :pending, :interaction_count)
+          '''
+    db.engine.execute(text(sql), **params)
+
+
+def add_journalist():
+    if random_bool():
+        otp_secret = random_chars(16, string.ascii_uppercase + '234567')
+    else:
+        otp_secret = None
+
+    is_totp = random_bool()
+    if is_totp:
+        hotp_counter = 0 if random_bool() else None
+    else:
+        hotp_counter = random.randint(0, 10000) if random_bool() else None
+
+    last_token = random_chars(6, string.digits) if random_bool() else None
+
+    params = {
+        'username': random_username(),
+        'pw_salt': random_bytes(1, 64, nullable=True),
+        'pw_hash': random_bytes(32, 64, nullable=True),
+        'is_admin': bool_or_none(),
+        'otp_secret': otp_secret,
+        'is_totp': is_totp,
+        'hotp_counter': hotp_counter,
+        'last_token': last_token,
+        'created_on': random_datetime(nullable=True),
+        'last_access': random_datetime(nullable=True),
+        'passphrase_hash': random_bytes(32, 64, nullable=True)
+    }
+    sql = '''INSERT INTO journalists (username, pw_salt, pw_hash,
+                is_admin, otp_secret, is_totp, hotp_counter, last_token,
+                created_on, last_access, passphrase_hash)
+             VALUES (:username, :pw_salt, :pw_hash, :is_admin,
+                :otp_secret, :is_totp, :hotp_counter, :last_token,
+                :created_on, :last_access, :passphrase_hash);
+          '''
+    db.engine.execute(text(sql), **params)
+
+
+class UpgradeTester():
+
+    '''This migration verifies that the deleted_by_source column now exists,
+    and that the data migration completed successfully.
+    '''
+
+    SOURCE_NUM = 200
+    JOURNO_NUM = 20
+
+    def __init__(self, config):
+        self.config = config
+        self.app = create_app(config)
+
+    def load_data(self):
+        with self.app.app_context():
+            for _ in range(self.JOURNO_NUM):
+                add_journalist()
+
+            add_source()
+
+            for jid in range(1, self.JOURNO_NUM):
+                self.add_reply(jid, 1)
+
+            db.session.commit()
+
+    @staticmethod
+    def add_reply(journalist_id, source_id):
+        params = {
+            'journalist_id': journalist_id,
+            'source_id': source_id,
+            'filename': random_chars(50),
+            'size': random.randint(0, 1024 * 1024 * 500),
+        }
+        sql = '''INSERT INTO replies (journalist_id, source_id, filename,
+                    size)
+                 VALUES (:journalist_id, :source_id, :filename, :size)
+              '''
+        db.engine.execute(text(sql), **params)
+
+    def check_upgrade(self):
+        with self.app.app_context():
+            replies = db.engine.execute(
+                text('SELECT * FROM replies')).fetchall()
+            assert len(replies) == self.JOURNO_NUM - 1
+
+            for reply in replies:
+                assert reply.deleted_by_source == False  # noqa
+
+
+class DowngradeTester():
+
+    SOURCE_NUM = 200
+    JOURNO_NUM = 20
+
+    def __init__(self, config):
+        self.config = config
+        self.app = create_app(config)
+
+    def load_data(self):
+        with self.app.app_context():
+            for _ in range(self.JOURNO_NUM):
+                add_journalist()
+
+            add_source()
+
+            for jid in range(1, self.JOURNO_NUM):
+                self.add_reply(jid, 1)
+
+            db.session.commit()
+
+    @staticmethod
+    def add_reply(journalist_id, source_id):
+        params = {
+            'journalist_id': journalist_id,
+            'source_id': source_id,
+            'filename': random_chars(50),
+            'size': random.randint(0, 1024 * 1024 * 500),
+            'deleted_by_source': False,
+        }
+        sql = '''INSERT INTO replies (journalist_id, source_id, filename,
+                    size, deleted_by_source)
+                 VALUES (:journalist_id, :source_id, :filename, :size,
+                    :deleted_by_source)
+              '''
+        db.engine.execute(text(sql), **params)
+
+    def check_downgrade(self):
+        '''Verify that the deleted_by_source column is now gone, and
+        otherwise the table has the expected number of rows.
+        '''
+        with self.app.app_context():
+            sql = "SELECT * FROM replies"
+            replies = db.engine.execute(text(sql)).fetchall()
+
+            for reply in replies:
+                try:
+                    # This should produce an exception, as the column (should)
+                    # be gone.
+                    assert reply['deleted_by_source'] is None
+                except NoSuchColumnError:
+                    pass
+
+            assert len(replies) == self.JOURNO_NUM - 1
diff --git a/securedrop/tests/test_integration.py b/securedrop/tests/test_integration.py
--- a/securedrop/tests/test_integration.py
+++ b/securedrop/tests/test_integration.py
@@ -441,11 +441,6 @@ def helper_test_reply(self, test_reply, expected_success=True):
                 self.assertEqual(resp.status_code, 200)
                 self.assertIn("Reply deleted", resp.data)
 
-                # Make sure the reply is deleted from the filesystem
-                utils.async.wait_for_assertion(
-                    lambda: self.assertFalse(os.path.exists(
-                        current_app.storage.path(filesystem_id, msgid))))
-
             app.get('/logout')
 
     @patch('source_app.main.async_genkey')
diff --git a/securedrop/tests/test_source.py b/securedrop/tests/test_source.py
--- a/securedrop/tests/test_source.py
+++ b/securedrop/tests/test_source.py
@@ -14,7 +14,7 @@
 import version
 
 from db import db
-from models import Source
+from models import Source, Reply
 from source_app import main as source_app_main
 from utils.db_helper import new_codename
 from utils.instrument import InstrumentedApp
@@ -394,6 +394,7 @@ def test_delete_all_successfully_deletes_replies(source_app):
     with source_app.app_context():
         journalist, _ = utils.db_helper.init_journalist()
         source, codename = utils.db_helper.init_source()
+        source_id = source.id
         utils.db_helper.reply(journalist, source, 1)
 
     with source_app.test_client() as app:
@@ -406,8 +407,41 @@ def test_delete_all_successfully_deletes_replies(source_app):
         text = resp.data.decode('utf-8')
         assert "All replies have been deleted" in text
 
+    with source_app.app_context():
+        source = Source.query.get(source_id)
+        replies = Reply.query.filter(Reply.source_id == source_id).all()
+        for reply in replies:
+            assert reply.deleted_by_source is True
+
+
+def test_delete_all_replies_deleted_by_source_but_not_journalist(source_app):
+    """Replies can be deleted by a source, but not by journalists. As such,
+    replies may still exist in the replies table, but no longer be visible."""
+    with source_app.app_context():
+        journalist, _ = utils.db_helper.init_journalist()
+        source, codename = utils.db_helper.init_source()
+        utils.db_helper.reply(journalist, source, 1)
+        replies = Reply.query.filter(Reply.source_id == source.id).all()
+        for reply in replies:
+            reply.deleted_by_source = True
+            db.session.add(reply)
+            db.session.commit()
+
+    with source_app.test_client() as app:
+        with patch.object(source_app.logger, 'error') as logger:
+            resp = app.post(url_for('main.login'),
+                            data=dict(codename=codename),
+                            follow_redirects=True)
+            assert resp.status_code == 200
+            resp = app.post(url_for('main.batch_delete'),
+                            follow_redirects=True)
+            assert resp.status_code == 200
+            logger.assert_called_once_with(
+                "Found no replies when at least one was expected"
+            )
+
 
-def test_delete_all_replies_already_deleted(source_app):
+def test_delete_all_replies_already_deleted_by_journalists(source_app):
     with source_app.app_context():
         journalist, _ = utils.db_helper.init_journalist()
         source, codename = utils.db_helper.init_source()