Repository: freedomofpress/securedrop

Issue:
Flask logging disappeared
At some point, running `python {source,journalist}.py` stopped printing the familiar (and useful) default Flask logging of requests, responses, and logger messages. We should get it back, it's very helpful for debugging.

Pull latest version of mat and verify the digital signature
The source requirements text file tells the SecureDrop installation process to download version 0.4.2 of mat and install it without ever verifying the digital signature. This process should ideally be updated to (1) download the latest version (0.5.2) and (2) verify the digital signature.



Discussion:

Resolving patch:
diff --git a/securedrop/source.py b/securedrop/source.py
--- a/securedrop/source.py
+++ b/securedrop/source.py
@@ -220,7 +220,6 @@ def normalize_timestamps(sid):
 def submit():
     msg = request.form['msg']
     fh = request.files['fh']
-    strip_metadata = True if 'notclean' in request.form else False
 
     fnames = []
     journalist_filename = g.source.journalist_filename()
@@ -234,7 +233,7 @@ def submit():
     if fh:
         g.source.interaction_count += 1
         fnames.append(store.save_file_submission(g.sid, g.source.interaction_count,
-            journalist_filename, fh.filename, fh.stream, fh.content_type, strip_metadata))
+            journalist_filename, fh.filename, fh.stream))
         flash("{} '{}'. {}".format(SUBMIT_DOC_NOTIFY_STR,
                                    fh.filename or '[unnamed]',
                                    SUBMIT_CODENAME_NOTIFY_STR), "notification")
diff --git a/securedrop/store.py b/securedrop/store.py
--- a/securedrop/store.py
+++ b/securedrop/store.py
@@ -8,10 +8,6 @@
 import tempfile
 import subprocess
 from cStringIO import StringIO
-from shutil import copyfileobj
-
-from MAT import mat
-from MAT import strippers
 
 import logging
 log = logging.getLogger(__name__)
@@ -79,13 +75,12 @@ def get_bulk_archive(filenames):
     return zip_file
 
 
-def save_file_submission(sid, count, journalist_filename, filename, stream, content_type, strip_metadata):
+def save_file_submission(sid, count, journalist_filename, filename, stream):
     sanitized_filename = secure_filename(filename)
-    clean_file = sanitize_metadata(stream, content_type, strip_metadata)
 
     s = StringIO()
     with zipfile.ZipFile(s, 'w') as zf:
-        zf.writestr(sanitized_filename, clean_file.read() if clean_file else stream.read())
+        zf.writestr(sanitized_filename, stream.read())
     s.reset()
 
     filename = "{0}-{1}-doc.zip.gpg".format(count, journalist_filename)
@@ -101,9 +96,8 @@ def save_message_submission(sid, count, journalist_filename, message):
     return filename
 
 
-def secure_unlink(fn, recursive=False, do_verify = True):
-    if do_verify:
-        verify(fn)
+def secure_unlink(fn, recursive=False):
+    verify(fn)
     command = ['srm']
     if recursive:
         command.append('-r')
@@ -113,35 +107,3 @@ def secure_unlink(fn, recursive=False, do_verify = True):
 
 def delete_source_directory(source_id):
     secure_unlink(path(source_id), recursive=True)
-
-def metadata_handler(f):
-    return mat.create_class_file(f, False, add2archive=True)
-
-def sanitize_metadata(stream, content_type, strip_metadata):
-    text_plain = content_type == 'text/plain'
-
-    s = None
-    t = None
-    clean_file = False
-
-    if strip_metadata and not text_plain:
-        t = tempfile.NamedTemporaryFile(delete = False)
-        copyfileobj(stream, t)
-        t.flush()
-        file_meta = metadata_handler(t.name)
-
-        if not file_meta.is_clean():
-            file_meta.remove_all()
-            f = open(t.name)
-            s = StringIO()
-            s.write(f.read())
-            f.close()
-            s.reset()
-            secure_unlink(t.name, do_verify = False)
-            t.close()
-        else:
-            secure_unlink(t.name, do_verify = False)
-            t.close()
-
-    return s
-


Test patch:
diff --git a/securedrop/tests/test_images/dirty.jpg b/securedrop/tests/test_images/dirty.jpg
deleted file mode 100644
Binary files a/securedrop/tests/test_images/dirty.jpg and /dev/null differ
diff --git a/securedrop/tests/test_unit.py b/securedrop/tests/test_unit.py
--- a/securedrop/tests/test_unit.py
+++ b/securedrop/tests/test_unit.py
@@ -26,6 +26,7 @@
 import test_setup
 from db import db_session, Source
 
+
 def _block_on_reply_keypair_gen(codename):
     sid = crypto_util.hash_codename(codename)
     while not crypto_util.getkey(sid):
@@ -193,81 +194,6 @@ def test_submit_both(self):
         self.assertIn(source.SUBMIT_MSG_NOTIFY_STR, rv.data)
         self.assertIn(escape("%s '%s'. %s" % (source.SUBMIT_DOC_NOTIFY_STR, 'test.txt', source.SUBMIT_CODENAME_NOTIFY_STR)), rv.data)
 
-    def test_submit_dirty_file_to_be_cleaned(self):
-        self.gpg = gnupg.GPG(homedir=config.GPG_KEY_DIR)
-        img = open(os.getcwd()+'/tests/test_images/dirty.jpg')
-        img_metadata = store.metadata_handler(img.name)
-        self.assertFalse(img_metadata.is_clean(), "The file is dirty.")
-        codename = self._new_codename()
-        rv = self.client.post('/submit', data=dict(
-            msg="",
-            fh=(img, 'dirty.jpg'),
-            notclean='True',
-        ), follow_redirects=True)
-        self.assertEqual(rv.status_code, 200)
-        self.assertIn(escape("%s '%s'. %s" % (source.SUBMIT_DOC_NOTIFY_STR, 'dirty.jpg', source.SUBMIT_CODENAME_NOTIFY_STR)), rv.data)
-
-        store_dirs = [os.path.join(config.STORE_DIR,d) for d in os.listdir(config.STORE_DIR) if os.path.isdir(os.path.join(config.STORE_DIR,d))]
-        latest_subdir = max(store_dirs, key=os.path.getmtime)
-        zip_gpg_files = [os.path.join(latest_subdir,f) for f in os.listdir(latest_subdir) if os.path.isfile(os.path.join(latest_subdir,f))]
-        self.assertEqual(len(zip_gpg_files), 1)
-        zip_gpg = zip_gpg_files[0]
-
-        zip_gpg_file = open(zip_gpg)
-        decrypted_data = self.gpg.decrypt_file(zip_gpg_file)
-        self.assertTrue(decrypted_data.ok, 'Checking the integrity of the data after decryption.')
-
-        s = StringIO(decrypted_data.data)
-        zip_file = zipfile.ZipFile(s, 'r')
-        clean_file = open(os.path.join(latest_subdir,'dirty.jpg'), 'w+b')
-        clean_file.write(zip_file.read('dirty.jpg'))
-        clean_file.seek(0)
-        zip_file.close()
-
-        # check for the actual file been clean
-        clean_file_metadata = store.metadata_handler(clean_file.name)
-        self.assertTrue(clean_file_metadata.is_clean(), "the file is now clean.")
-        zip_gpg_file.close()
-        clean_file.close()
-        img.close()
-
-    def test_submit_dirty_file_to_not_clean(self):
-        self.gpg = gnupg.GPG(homedir=config.GPG_KEY_DIR)
-        img = open(os.getcwd()+'/tests/test_images/dirty.jpg')
-        img_metadata = store.metadata_handler(img.name)
-        self.assertFalse(img_metadata.is_clean(), "The file is dirty.")
-        codename = self._new_codename()
-        rv = self.client.post('/submit', data=dict(
-            msg="",
-            fh=(img, 'dirty.jpg'),
-        ), follow_redirects=True)
-        self.assertEqual(rv.status_code, 200)
-        self.assertIn(escape("%s '%s'. %s" % (source.SUBMIT_DOC_NOTIFY_STR, 'dirty.jpg', source.SUBMIT_CODENAME_NOTIFY_STR)), rv.data)
-
-        store_dirs = [os.path.join(config.STORE_DIR,d) for d in os.listdir(config.STORE_DIR) if os.path.isdir(os.path.join(config.STORE_DIR,d))]
-        latest_subdir = max(store_dirs, key=os.path.getmtime)
-        zip_gpg_files = [os.path.join(latest_subdir,f) for f in os.listdir(latest_subdir) if os.path.isfile(os.path.join(latest_subdir,f))]
-        self.assertEqual(len(zip_gpg_files), 1)
-        zip_gpg = zip_gpg_files[0]
-
-        zip_gpg_file = open(zip_gpg)
-        decrypted_data = self.gpg.decrypt_file(zip_gpg_file)
-        self.assertTrue(decrypted_data.ok, 'Checking the integrity of the data after decryption.')
-
-        s = StringIO(decrypted_data.data)
-        zip_file = zipfile.ZipFile(s, 'r')
-        clean_file = open(os.path.join(latest_subdir,'dirty.jpg'), 'w+b')
-        clean_file.write(zip_file.read('dirty.jpg'))
-        clean_file.seek(0)
-        zip_file.close()
-
-        # check for the actual file been clean
-        clean_file_metadata = store.metadata_handler(clean_file.name)
-        self.assertFalse(clean_file_metadata.is_clean(), "the file is was not cleaned.")
-        zip_gpg_file.close()
-        clean_file.close()
-        img.close()
-
     def test_submit_clean_file(self):
         img = open(os.getcwd()+'/tests/test_images/clean.jpg')
         codename = self._new_codename()