Repository: freedomofpress/securedrop

Issue:
Timestamps
The issue of having timestamps on submitted files, and the disadvantages of retaining that metadata which can be correlated with other known variables, has been brought up.

One of the solutions which has been posited is scrambling/randomizing or nullifying the timestamps, but the problem with that is journalists would lose track of the order in which submissions arrived. With document flagging implemented in the future (#124) that would be less of an issue.

I believe @Hainish proposed an idea that would employ cryptographic hashes to retain the ordering of documents, while ditching the timestamps, but I don't know much about the specifics. Would love to hear more. So far I've done a preliminary search for possible filesystem/kernel or mount option-based solutions and didn't come up with anything.



Discussion:
I have a nearly-complete rewrite of the database code that stores metadata in the db rather than stat'ing it from the filesystem (which is less efficient anyway).

In that rewrite, the order of submissions was retained through the natural ordering of an auto-incrementing primary key, and only the timestamp of the last upload ("last updated") was stored in the db. This retained a useful order of submissions/collections for the journalist interface while minimizing stored metadata. Of course, there is still metadata in the filesystem. Linux doesn't store file creation times. `stat.st_ctime`, which we use now, is platform dependent, and returns the last time of metadata change on Linux. So, we could minimize metadata in the filesystem with this procedure:
1. On a new submission, update the database. Source table has a `last_submitted` datetime field. Submission table primary keys are used to keep submissions in "sort order", and the most recent submission for a given source is correlated with `last_submitted`.
2. `touch` the other submissions in the source's directory. This will update their metadata to the same time as the latest submission, so we don't leak anything other than that.


Resolving patch:
diff --git a/securedrop/source.py b/securedrop/source.py
--- a/securedrop/source.py
+++ b/securedrop/source.py
@@ -5,6 +5,7 @@
 from functools import wraps
 import zipfile
 from cStringIO import StringIO
+import subprocess
 
 import logging
 # This module's logger is explicitly labeled so the correct logger is used,
@@ -180,6 +181,22 @@ def async_genkey(sid, codename):
             flagged=g.source.flagged, haskey=crypto_util.getkey(g.sid))
 
 
+def normalize_timestamps(sid):
+    """
+    Update the timestamps on all of the source's submissions to match that of
+    the latest submission. This minimizes metadata that could be useful to
+    investigators. See #301.
+    """
+    sub_paths = [ store.path(sid, submission.filename)
+                  for submission in g.source.submissions ]
+    if len(sub_paths) > 1:
+        args = ["touch"]
+        args.extend(sub_paths[:-1])
+        rc = subprocess.call(args)
+        if rc != 0:
+            app.logger.warning("Couldn't normalize submission timestamps (touch exited with %d)" % rc)
+
+
 @app.route('/submit', methods=('POST',))
 @login_required
 def submit():
@@ -204,6 +221,7 @@ def submit():
 
     g.source.last_updated = datetime.now()
     db_session.commit()
+    normalize_timestamps(g.sid)
 
     return redirect(url_for('lookup'))
 


Test patch:
diff --git a/securedrop/tests/unit_tests.py b/securedrop/tests/unit_tests.py
--- a/securedrop/tests/unit_tests.py
+++ b/securedrop/tests/unit_tests.py
@@ -197,22 +197,21 @@ def test_submit_dirty_file_to_be_cleaned(self):
         img = open(os.getcwd()+'/tests/test_images/dirty.jpg')
         img_metadata = store.metadata_handler(img.name)
         self.assertFalse(img_metadata.is_clean(), "The file is dirty.")
-        del(img_metadata)
         codename = self._new_codename()
         rv = self.client.post('/submit', data=dict(
-            msg="This is a test",
+            msg="",
             fh=(img, 'dirty.jpg'),
             notclean='True',
         ), follow_redirects=True)
         self.assertEqual(rv.status_code, 200)
-        self.assertIn("Thanks! We received your message.", rv.data)
         self.assertIn(escape("Thanks! We received your document 'dirty.jpg'."),
                       rv.data)
 
         store_dirs = [os.path.join(config.STORE_DIR,d) for d in os.listdir(config.STORE_DIR) if os.path.isdir(os.path.join(config.STORE_DIR,d))]
         latest_subdir = max(store_dirs, key=os.path.getmtime)
         zip_gpg_files = [os.path.join(latest_subdir,f) for f in os.listdir(latest_subdir) if os.path.isfile(os.path.join(latest_subdir,f))]
-        zip_gpg = max(zip_gpg_files, key=os.path.getmtime)
+        self.assertEqual(len(zip_gpg_files), 1)
+        zip_gpg = zip_gpg_files[0]
 
         zip_gpg_file = open(zip_gpg)
         decrypted_data = self.gpg.decrypt_file(zip_gpg_file)
@@ -228,7 +227,6 @@ def test_submit_dirty_file_to_be_cleaned(self):
         # check for the actual file been clean
         clean_file_metadata = store.metadata_handler(clean_file.name)
         self.assertTrue(clean_file_metadata.is_clean(), "the file is now clean.")
-        del(clean_file_metadata)
         zip_gpg_file.close()
         clean_file.close()
         img.close()
@@ -238,21 +236,20 @@ def test_submit_dirty_file_to_not_clean(self):
         img = open(os.getcwd()+'/tests/test_images/dirty.jpg')
         img_metadata = store.metadata_handler(img.name)
         self.assertFalse(img_metadata.is_clean(), "The file is dirty.")
-        del(img_metadata)
         codename = self._new_codename()
         rv = self.client.post('/submit', data=dict(
-            msg="This is a test",
+            msg="",
             fh=(img, 'dirty.jpg'),
         ), follow_redirects=True)
         self.assertEqual(rv.status_code, 200)
-        self.assertIn("Thanks! We received your message.", rv.data)
         self.assertIn(escape("Thanks! We received your document 'dirty.jpg'."),
                       rv.data)
 
         store_dirs = [os.path.join(config.STORE_DIR,d) for d in os.listdir(config.STORE_DIR) if os.path.isdir(os.path.join(config.STORE_DIR,d))]
         latest_subdir = max(store_dirs, key=os.path.getmtime)
         zip_gpg_files = [os.path.join(latest_subdir,f) for f in os.listdir(latest_subdir) if os.path.isfile(os.path.join(latest_subdir,f))]
-        zip_gpg = max(zip_gpg_files, key=os.path.getmtime)
+        self.assertEqual(len(zip_gpg_files), 1)
+        zip_gpg = zip_gpg_files[0]
 
         zip_gpg_file = open(zip_gpg)
         decrypted_data = self.gpg.decrypt_file(zip_gpg_file)
@@ -268,7 +265,6 @@ def test_submit_dirty_file_to_not_clean(self):
         # check for the actual file been clean
         clean_file_metadata = store.metadata_handler(clean_file.name)
         self.assertFalse(clean_file_metadata.is_clean(), "the file is was not cleaned.")
-        del(clean_file_metadata)
         zip_gpg_file.close()
         clean_file.close()
         img.close()