Repository: freedomofpress/securedrop

Issue:
Create securedrop-app-code Debian package with Python 3 dependencies
Now that we have basic Python 3 compatibility (#997, resolved through #4239), we need to ensure that all the `securedrop-app-code` dependencies transition to Python 3 as well. The plan of record is to use `dh-virtualenv` to ship these dependencies in the package itself, instead of depending on the system Python libraries. 

This will allow us to use consistent dependency management processes for SecureDrop Workstation and SecureDrop core, and will get us closer to resolving #3407.


Discussion:
For the 4/17-5/1 sprint, Kushal will prepare at least a draft PR for broader team input, which will not use FPF's PyPI mirror yet.
hey @kushaldas what is the status of this? 
can you push up whatever wip you have when you get a chance @kushaldas ? 
I did not manage to work on this enough during that sprint, but, I have other WIP work on my backup system (a `make clean` removed from my laptop during the last trip). I will push it here with details of things needs to be done. 
please push up whatever wip you have locally so @rmol can check it out since y'all will be working together next sprint to get the combined `mod_wsgi` changes and this issue merged together. 

Resolving patch:
diff --git a/securedrop/i18n_tool.py b/securedrop/i18n_tool.py
--- a/securedrop/i18n_tool.py
+++ b/securedrop/i18n_tool.py
@@ -4,7 +4,6 @@
 from __future__ import print_function
 import argparse
 import io
-import six
 import logging
 import os
 import glob
@@ -61,7 +60,7 @@ def file_is_modified(self, path):
 
     def ensure_i18n_remote(self, args):
         k = {'_cwd': args.root}
-        if six.b('i18n') not in git.remote(**k).stdout:
+        if b'i18n' not in git.remote(**k).stdout:
             git.remote.add('i18n', args.url, **k)
         git.fetch('i18n', **k)
 
diff --git a/securedrop/journalist_app/__init__.py b/securedrop/journalist_app/__init__.py
--- a/securedrop/journalist_app/__init__.py
+++ b/securedrop/journalist_app/__init__.py
@@ -7,6 +7,7 @@
 from flask_babel import gettext
 from flask_wtf.csrf import CSRFProtect, CSRFError
 from os import path
+import sys
 from werkzeug.exceptions import default_exceptions
 
 import i18n
@@ -141,6 +142,17 @@ def setup_g():
                                       'SESSION_EXPIRATION_MINUTES',
                                       120))
 
+        # Work around https://github.com/lepture/flask-wtf/issues/275
+        # -- after upgrading from Python 2 to Python 3, any existing
+        # session's csrf_token value will be retrieved as bytes,
+        # causing a TypeError. This simple fix, deleting the existing
+        # token, was suggested in the issue comments. This code will
+        # be safe to remove after Python 2 reaches EOL in 2020, and no
+        # supported SecureDrop installations can still have this
+        # problem.
+        if sys.version_info.major > 2 and type(session.get('csrf_token')) is bytes:
+            del session['csrf_token']
+
         uid = session.get('uid', None)
         if uid:
             g.user = Journalist.query.get(uid)
diff --git a/setup.py b/setup.py
new file mode 100644
--- /dev/null
+++ b/setup.py
@@ -0,0 +1,24 @@
+import setuptools
+
+long_description = "The SecureDrop whistleblower platform."
+
+setuptools.setup(
+    name="securedrop-app-code",
+    version="1.0.0~rc1",
+    author="Freedom of the Press Foundation",
+    author_email="securedrop@freedom.press",
+    description="SecureDrop Server",
+    long_description=long_description,
+    long_description_content_type="text/markdown",
+    license="GPLv3+",
+    python_requires=">=3.5",
+    url="https://github.com/freedomofpress/securedrop",
+    classifiers=(
+        "Development Status :: 5 - Stable",
+        "Programming Language :: Python :: 3",
+        "Topic :: Software Development :: Libraries :: Python Modules",
+        "License :: OSI Approved :: GNU General Public License v3 or later (GPLv3+)",
+        "Intended Audience :: Developers",
+        "Operating System :: OS Independent",
+    ),
+)


Test patch:
diff --git a/molecule/builder-xenial/tests/test_build_dependencies.py b/molecule/builder-xenial/tests/test_build_dependencies.py
--- a/molecule/builder-xenial/tests/test_build_dependencies.py
+++ b/molecule/builder-xenial/tests/test_build_dependencies.py
@@ -8,16 +8,6 @@
 ]
 
 
-def test_pip_wheel_installed(host):
-    """
-    Ensure `wheel` is installed via pip, for packaging Python
-    dependencies into a Debian package.
-    """
-    c = host.run("pip list installed")
-    assert "wheel" in c.stdout
-    assert c.rc == 0
-
-
 def test_sass_gem_installed(host):
     """
     Ensure the `sass` Ruby gem is installed, for compiling SASS to CSS.
@@ -31,7 +21,7 @@ def test_pip_dependencies_installed(host):
     """
     Ensure the development pip dependencies are installed
     """
-    c = host.run("pip list installed")
+    c = host.run("pip3 list installed")
     assert "Flask-Babel" in c.stdout
     assert c.rc == 0
 
diff --git a/molecule/builder-xenial/tests/test_securedrop_deb_package.py b/molecule/builder-xenial/tests/test_securedrop_deb_package.py
--- a/molecule/builder-xenial/tests/test_securedrop_deb_package.py
+++ b/molecule/builder-xenial/tests/test_securedrop_deb_package.py
@@ -164,7 +164,7 @@ def test_deb_package_contains_no_config_file(host, deb):
     deb_package = host.file(deb.format(
         securedrop_test_vars.securedrop_version))
     c = host.run("dpkg-deb --contents {}".format(deb_package.path))
-    assert not re.search(r"^.*/config\.py$", c.stdout, re.M)
+    assert not re.search(r"^ ./var/www/securedrop/config.py$", c.stdout, re.M)
 
 
 @pytest.mark.parametrize("deb", deb_packages)
@@ -198,9 +198,9 @@ def test_deb_package_contains_mo_file(host, deb):
 @pytest.mark.parametrize("deb", deb_packages)
 def test_deb_package_contains_no_generated_assets(host, deb):
     """
-    Ensures the `securedrop-app-code` package does not ship a minified
-    static assets, which are built automatically via Flask-Assets, and may be
-    present in the source directory used to build from.
+    Ensures the `securedrop-app-code` package does not ship minified
+    static assets, which are built automatically via Flask-Assets, and
+    may be present in the source directory used to build from.
     """
     deb_package = host.file(deb.format(
         securedrop_test_vars.securedrop_version))
@@ -223,7 +223,7 @@ def test_deb_package_contains_no_generated_assets(host, deb):
                              "/static/.webassets-cache/.+$", c.stdout, re.M)
 
         # no SASS files should exist; only the generated CSS files.
-        assert not re.search("^.*sass.*$", c.stdout, re.M)
+        assert not re.search("^.*sass$", c.stdout, re.M)
 
         # no .map files should exist; only the generated CSS files.
         assert not re.search("^.*css.map$", c.stdout, re.M)
diff --git a/molecule/testinfra/staging/app-code/test_redis_worker.py b/molecule/testinfra/staging/app-code/test_redis_worker.py
--- a/molecule/testinfra/staging/app-code/test_redis_worker.py
+++ b/molecule/testinfra/staging/app-code/test_redis_worker.py
@@ -8,7 +8,7 @@
 
 @pytest.mark.parametrize('config_line', [
   '[program:securedrop_worker]',
-  'command=/usr/local/bin/rqworker',
+  'command=/opt/venvs/securedrop-app-code/bin/rqworker',
   "directory={}".format(securedrop_test_vars.securedrop_code),
   'autostart=true',
   'autorestart=true',
@@ -16,7 +16,6 @@
   'stderr_logfile=/var/log/securedrop_worker/err.log',
   'stdout_logfile=/var/log/securedrop_worker/out.log',
   "user={}".format(securedrop_test_vars.securedrop_user),
-  'environment=HOME="/tmp/python-gnupg"',
 ])
 def test_redis_worker_configuration(host, config_line):
     """
diff --git a/molecule/testinfra/staging/app-code/test_securedrop_app_code.py b/molecule/testinfra/staging/app-code/test_securedrop_app_code.py
--- a/molecule/testinfra/staging/app-code/test_securedrop_app_code.py
+++ b/molecule/testinfra/staging/app-code/test_securedrop_app_code.py
@@ -45,7 +45,7 @@ def test_securedrop_application_test_locale(host):
         assert securedrop_config.contains("^DEFAULT_LOCALE")
         assert securedrop_config.content_string.count("DEFAULT_LOCALE") == 1
         assert securedrop_config.content_string.count("SUPPORTED_LOCALES") == 1
-        assert "\nSUPPORTED_LOCALES = [u'el', u'ar', 'en_US']\n" in securedrop_config.content_string
+        assert "\nSUPPORTED_LOCALES = ['el', 'ar', 'en_US']\n" in securedrop_config.content_string
 
 
 def test_securedrop_application_test_journalist_key(host):
diff --git a/molecule/testinfra/staging/app/apache/test_apache_system_config.py b/molecule/testinfra/staging/app/apache/test_apache_system_config.py
--- a/molecule/testinfra/staging/app/apache/test_apache_system_config.py
+++ b/molecule/testinfra/staging/app/apache/test_apache_system_config.py
@@ -6,7 +6,6 @@
 
 
 @pytest.mark.parametrize("package", [
-    "libapache2-mod-wsgi",
     "libapache2-mod-xsendfile",
 ])
 def test_apache_apt_packages(host, package):
diff --git a/molecule/testinfra/staging/app/test_appenv.py b/molecule/testinfra/staging/app/test_appenv.py
--- a/molecule/testinfra/staging/app/test_appenv.py
+++ b/molecule/testinfra/staging/app/test_appenv.py
@@ -1,13 +1,16 @@
+import os.path
 import pytest
 
 testinfra_hosts = ["app-staging"]
 sdvars = pytest.securedrop_test_vars
 
+sdbin = "/opt/venvs/securedrop-app-code/bin"
+
 
 @pytest.mark.parametrize('exp_pip_pkg', sdvars.pip_deps)
 def test_app_pip_deps(host, exp_pip_pkg):
     """ Ensure pip dependencies are installed """
-    pip = host.pip_package.get_packages()
+    pip = host.pip_package.get_packages(pip_path=os.path.join(sdbin, "pip"))
     assert pip[exp_pip_pkg['name']]['version'] == exp_pip_pkg['version']
 
 
@@ -77,4 +80,4 @@ def test_app_workerlog_dir(host):
         assert f.is_directory
         assert f.user == "root"
         assert f.group == "root"
-        assert f.mode == 0o644
+        assert f.mode == 0o700