Repository: freedomofpress/securedrop

Issue:
Journalists should be able to bulk download from document server
Right now, journalists can only download one file at a time, even if there are dozens new submissions in any given session. The New Yorker team asked if we can enable bulk downloading so that journalists can download multiple files at once. 



Discussion:
looking at this now

Check out #131, which adds checkboxes next to documents in a collection so they can be deleted. You can probably reuse this code, adding a "download" button.

FYI @msjoinder, @jacksingleton's comment above was meant to mean "I'm taking this to work on at the hackathon". I'll wait for his pull request, and then compare the two and figure out what to merge. Sorry that wasn't more clear!


Resolving patch:
diff --git a/modules/deaddrop/files/deaddrop/journalist.py b/modules/deaddrop/files/deaddrop/journalist.py
--- a/modules/deaddrop/files/deaddrop/journalist.py
+++ b/modules/deaddrop/files/deaddrop/journalist.py
@@ -71,11 +71,22 @@ def reply():
     store.path(sid, 'reply-%s.gpg' % uuid.uuid4()))
   return render_template('reply.html', sid=sid, codename=crypto.displayid(sid))
 
-@app.route('/delete', methods=('POST',))
-def delete():
+@app.route('/bulk', methods=('POST',))
+def bulk():
+  action = request.form['action']
+
   sid = request.form['sid']
   doc_names_selected = request.form.getlist('doc_names_selected')
-  docs_selected = [doc for doc in get_docs(sid) if doc['name'] in doc_names_selected]
+  docs_selected = [ doc for doc in get_docs(sid) if doc['name'] in doc_names_selected ]
+
+  if action == 'download':
+    return bulk_download(sid, docs_selected)
+  elif action == 'delete':
+    return bulk_delete(sid, docs_selected)
+  else:
+    abort(422)
+
+def bulk_delete(sid, docs_selected):
   confirm_delete = bool(request.form.get('confirm_delete', False))
   if confirm_delete:
       for doc in docs_selected:
@@ -84,6 +95,13 @@ def delete():
   return render_template('delete.html', sid=sid, codename=crypto.displayid(sid),
                          docs_selected=docs_selected, confirm_delete=confirm_delete)
 
+def bulk_download(sid, docs_selected):
+  filenames = [store.path(sid, doc['name']) for doc in docs_selected]
+  zip = store.get_bulk_archive(filenames)
+  return send_file(zip, mimetype="application/zip",  attachment_filename=crypto.displayid(sid), as_attachment=True)
+
+
+  
 if __name__ == "__main__":
   # TODO: make sure this gets run by the web server
   CsrfProtect(app)
diff --git a/modules/deaddrop/files/deaddrop/store.py b/modules/deaddrop/files/deaddrop/store.py
--- a/modules/deaddrop/files/deaddrop/store.py
+++ b/modules/deaddrop/files/deaddrop/store.py
@@ -2,6 +2,9 @@
 import os
 import re
 import config
+import zipfile
+import crypto
+import uuid
 
 VALIDATE_FILENAME = re.compile("^(reply-)?[a-f0-9-]+(_msg|_doc|)\.gpg$").match
 
@@ -48,5 +51,14 @@ def path(*s):
     verify(absolute)
     return absolute
 
+def get_bulk_archive(filenames):
+    zip_file_name = os.path.join(config.TEMP_DIR, str(uuid.uuid4()) + '.zip')
+    with zipfile.ZipFile(zip_file_name, 'w') as zip:
+        for filename in filenames:
+            verify(filename)
+            basename = os.path.basename(filename)
+            zip.write(filename, arcname=basename)
+    return zip_file_name
+
 def log(msg):
     file(path('NOTES'), 'a').write(msg)


Test patch:
diff --git a/modules/deaddrop/files/deaddrop/test.py b/modules/deaddrop/files/deaddrop/test.py
--- a/modules/deaddrop/files/deaddrop/test.py
+++ b/modules/deaddrop/files/deaddrop/test.py
@@ -19,7 +19,7 @@
 
 # Set the environment variable so config.py uses a test environment
 os.environ['DEADDROPENV'] = 'test'
-import config, crypto
+import config, crypto, store
 
 import source, store, journalist
 
@@ -27,10 +27,21 @@ def _block_on_reply_keypair_gen(codename):
     sid = crypto.shash(codename)
     while not crypto.getkey(sid): sleep(0.1)
 
+
+def _setup_test_docs(sid, files):
+    filenames = [ os.path.join(config.STORE_DIR, sid, file) for file in files ]
+    for file in filenames:
+        dirname = os.path.dirname(file)
+        if not os.path.exists(dirname):
+            os.makedirs(dirname)
+        with open(file, 'w') as myfile:
+            myfile.write('test')
+    return filenames
+
 def shared_setup():
     """Set up the file system and GPG"""
     # Create directories for the file store and the GPG keyring
-    for d in (config.TEST_DIR, config.STORE_DIR, config.GPG_KEY_DIR):
+    for d in (config.TEST_DIR, config.STORE_DIR, config.GPG_KEY_DIR, config.TEMP_DIR):
         try:
             os.mkdir(d)
         except OSError:
@@ -189,12 +200,10 @@ def test_tor2web_warning(self):
 
 class TestJournalist(TestCase):
 
-    @classmethod
-    def setUpClass(cls):
+    def setUp(cls):
         shared_setup()
 
-    @classmethod
-    def tearDownClass(cls):
+    def tearDown(cls):
         shared_teardown()
 
     def create_app(self):
@@ -212,6 +221,19 @@ def test_index(self):
         self.assertIn("Latest submissions", rv.data)
         self.assertIn("No documents have been submitted!", rv.data)
 
+    def test_bulk_download(self):
+        sid = 'EQZGCJBRGISGOTC2NZVWG6LILJBHEV3CINNEWSCLLFTUWZJPKJFECLS2NZ4G4U3QOZCFKTTPNZMVIWDCJBBHMUDBGFHXCQ3R'
+        files = ['abc1_msg.gpg', 'abc2_msg.gpg']
+        filenames = _setup_test_docs(sid, files)
+
+        rv = self.client.post('/bulk', data=dict(
+            action='download',
+            sid=sid,
+            doc_names_selected=filenames
+        ))
+
+        self.assertEqual(rv.status_code, 200)
+        
 class TestIntegration(unittest.TestCase):
 
     def setUp(self):
@@ -270,7 +292,8 @@ def test_submit_message(self):
         self.assertEqual(rv.status_code, 200)
         soup = BeautifulSoup(rv.data)
         doc_name = soup.select('ul > li > input[name="doc_names_selected"]')[0]['value']
-        rv = self.journalist_app.post('/delete', data=dict(
+        rv = self.journalist_app.post('/bulk', data=dict(
+            action='delete',
             sid=sid,
             doc_names_selected=doc_name
         ))
@@ -282,7 +305,8 @@ def test_submit_message(self):
         # confirm delete submission
         doc_name = soup.select
         doc_name = soup.select('ul > li > input[name="doc_names_selected"]')[0]['value']
-        rv = self.journalist_app.post('/delete', data=dict(
+        rv = self.journalist_app.post('/bulk', data=dict(
+            action='delete',
             sid=sid,
             doc_names_selected=doc_name,
             confirm_delete="1"
@@ -340,7 +364,8 @@ def test_submit_file(self):
         self.assertEqual(rv.status_code, 200)
         soup = BeautifulSoup(rv.data)
         doc_name = soup.select('ul > li > input[name="doc_names_selected"]')[0]['value']
-        rv = self.journalist_app.post('/delete', data=dict(
+        rv = self.journalist_app.post('/bulk', data=dict(
+            action='delete',
             sid=sid,
             doc_names_selected=doc_name
         ))
@@ -352,7 +377,8 @@ def test_submit_file(self):
         # confirm delete submission
         doc_name = soup.select
         doc_name = soup.select('ul > li > input[name="doc_names_selected"]')[0]['value']
-        rv = self.journalist_app.post('/delete', data=dict(
+        rv = self.journalist_app.post('/bulk', data=dict(
+            action='delete',
             sid=sid,
             doc_names_selected=doc_name,
             confirm_delete="1"
@@ -419,10 +445,28 @@ def test_reply(self):
 
 class TestStore(unittest.TestCase):
     '''The set of tests for store.py.'''
+    @classmethod
+    def setUp(self):
+        shared_setup()
+
+    @classmethod
+    def tearDown(self):
+        shared_teardown()
+
     def test_verify(self):
         with self.assertRaises(store.PathException):
             store.verify(os.path.join(config.STORE_DIR, '..', 'etc', 'passwd'))
 
+    def test_get_zip(self):
+        sid = 'EQZGCJBRGISGOTC2NZVWG6LILJBHEV3CINNEWSCLLFTUWZJPKJFECLS2NZ4G4U3QOZCFKTTPNZMVIWDCJBBHMUDBGFHXCQ3R'
+        files = ['abc1_msg.gpg', 'abc2_msg.gpg']
+        filenames = _setup_test_docs(sid, files)
+
+        zip = store.get_bulk_archive(filenames)
+
+        zipfile_contents = zipfile.ZipFile(zip).namelist()
+        for file in files:
+            self.assertIn(file, zipfile_contents)
 
 if __name__ == "__main__":
     unittest.main()