Repository: freedomofpress/securedrop

Issue:
`make update-admin-pip-requirements` target fails
## Description

Trying to update the requirements*.txt files for the onion v3 branch.


## Steps to Reproduce

```
make update-admin-pip-requirements
```

## Expected Behavior

The command should not error out and update the dependencies as required.

## Actual Behavior

```
 $ make update-admin-pip-requirements
███ Updating admin pip requirements...
Run with DOCKER_BUILD_VERBOSE=true for more information
Docker image build in progress ..................................................................................... done !
Traceback (most recent call last):
  File "/opt/.venv/bin/pip-compile", line 10, in <module>
    sys.exit(cli())
  File "/opt/.venv/local/lib/python2.7/site-packages/click/core.py", line 722, in __call__
    return self.main(*args, **kwargs)
  File "/opt/.venv/local/lib/python2.7/site-packages/click/core.py", line 697, in main
    rv = self.invoke(ctx)
  File "/opt/.venv/local/lib/python2.7/site-packages/click/core.py", line 895, in invoke
    return ctx.invoke(self.callback, **ctx.params)
  File "/opt/.venv/local/lib/python2.7/site-packages/click/core.py", line 535, in invoke
    return callback(*args, **kwargs)
  File "/opt/.venv/local/lib/python2.7/site-packages/piptools/scripts/compile.py", line 129, in cli
    repository = PyPIRepository(pip_options, session, build_isolation)
  File "/opt/.venv/local/lib/python2.7/site-packages/piptools/repositories/pypi.py", line 73, in __init__
    self.finder = PackageFinder(**finder_kwargs)
TypeError: __init__() got an unexpected keyword argument 'index_urls'
make[1]: *** [Makefile:10: update-pip-requirements] Error 1
make: *** [Makefile:32: update-admin-pip-requirements] Error 2

```




Discussion:

Resolving patch:
diff --git a/admin/securedrop_admin/__init__.py b/admin/securedrop_admin/__init__.py
--- a/admin/securedrop_admin/__init__.py
+++ b/admin/securedrop_admin/__init__.py
@@ -33,10 +33,14 @@
 import subprocess
 import sys
 import types
+import json
+import base64
 import prompt_toolkit
 from prompt_toolkit.validation import Validator, ValidationError
 import yaml
 from pkg_resources import parse_version
+from cryptography.hazmat.primitives import serialization
+from cryptography.hazmat.primitives.asymmetric import x25519
 
 sdlog = logging.getLogger(__name__)
 RELEASE_KEY = '22245C81E3BAEB4138B36061310F561200F4AD77'
@@ -566,6 +570,73 @@ def sdconfig(args):
     return 0
 
 
+def generate_new_v3_keys():
+    """This function generate new keys for Tor v3 onion
+    services and returns them as as tuple.
+
+    :returns: Tuple(public_key, private_key)
+    """
+
+    private_key = x25519.X25519PrivateKey.generate()
+    private_bytes = private_key.private_bytes(
+        encoding=serialization.Encoding.Raw	,
+        format=serialization.PrivateFormat.Raw,
+        encryption_algorithm=serialization.NoEncryption())
+    public_key = private_key.public_key()
+    public_bytes = public_key.public_bytes(
+        encoding=serialization.Encoding.Raw,
+        format=serialization.PublicFormat.Raw)
+
+    # Base32 encode and remove base32 padding characters (`=`)
+    # Using try/except blocks for Python 2/3 support.
+    try:
+        public = base64.b32encode(public_bytes).replace('=', '') \
+                       .decode("utf-8")
+    except TypeError:
+        public = base64.b32encode(public_bytes).replace(b'=', b'') \
+                       .decode("utf-8")
+    try:
+        private = base64.b32encode(private_bytes).replace('=', '') \
+                        .decode("utf-8")
+    except TypeError:
+        private = base64.b32encode(private_bytes).replace(b'=', b'') \
+                        .decode("utf-8")
+    return public, private
+
+
+def find_or_generate_new_torv3_keys(args):
+    """
+    This method will either read v3 Tor onion service keys if found or generate
+    a new public/private keypair.
+    """
+    secret_key_path = os.path.join(args.ansible_path,
+                                   "tor_v3_keys.json")
+    if os.path.exists(secret_key_path):
+        print('Tor v3 onion service keys already exist in: {}'.format(
+            secret_key_path))
+        return 0
+    # No old keys, generate and store them first
+    app_journalist_public_key, \
+        app_journalist_private_key = generate_new_v3_keys()
+    # For app ssh service
+    app_ssh_public_key, app_ssh_private_key = generate_new_v3_keys()
+    # For mon ssh service
+    mon_ssh_public_key, mon_ssh_private_key = generate_new_v3_keys()
+    tor_v3_service_info = {
+            "app_journalist_public_key": app_journalist_public_key,
+            "app_journalist_private_key": app_journalist_private_key,
+            "app_ssh_public_key": app_ssh_public_key,
+            "app_ssh_private_key": app_ssh_private_key,
+            "mon_ssh_public_key": mon_ssh_public_key,
+            "mon_ssh_private_key": mon_ssh_private_key,
+    }
+    with open(secret_key_path, 'w') as fobj:
+        json.dump(tor_v3_service_info, fobj, indent=4)
+    print('Tor v3 onion service keys generated and stored in: {}'.format(
+        secret_key_path))
+    return 0
+
+
 def install_securedrop(args):
     """Install/Update SecureDrop"""
     SiteConfig(args).load()
@@ -827,6 +898,11 @@ class ArgParseFormatterCombo(argparse.ArgumentDefaultsHelpFormatter,
                                               help=run_tails_config.__doc__)
     parse_tailsconfig.set_defaults(func=run_tails_config)
 
+    parse_generate_tor_keys = subparsers.add_parser(
+        'generate_v3_keys',
+        help=find_or_generate_new_torv3_keys.__doc__)
+    parse_generate_tor_keys.set_defaults(func=find_or_generate_new_torv3_keys)
+
     parse_backup = subparsers.add_parser('backup',
                                          help=backup_securedrop.__doc__)
     parse_backup.set_defaults(func=backup_securedrop)


Test patch:
diff --git a/admin/tests/test_securedrop-admin.py b/admin/tests/test_securedrop-admin.py
--- a/admin/tests/test_securedrop-admin.py
+++ b/admin/tests/test_securedrop-admin.py
@@ -22,6 +22,7 @@
 import argparse
 from flaky import flaky
 from os.path import dirname, join, basename, exists
+import json
 import mock
 from prompt_toolkit.validation import ValidationError
 import pytest
@@ -1008,3 +1009,58 @@ def test_load(self, caplog):
         with pytest.raises(yaml.YAMLError) as e:
             site_config.load()
         assert 'issue processing' in caplog.text
+
+
+def test_generate_new_v3_keys():
+    public, private = securedrop_admin.generate_new_v3_keys()
+
+    for key in [public, private]:
+        # base32 padding characters should be removed
+        assert '=' not in key
+        assert len(key) == 52
+
+
+def test_find_or_generate_new_torv3_keys_first_run(tmpdir, capsys):
+    args = argparse.Namespace(ansible_path=str(tmpdir))
+
+    return_code = securedrop_admin.find_or_generate_new_torv3_keys(args)
+
+    captured = capsys.readouterr()
+    assert 'Tor v3 onion service keys generated' in captured.out
+    assert return_code == 0
+
+    secret_key_path = os.path.join(args.ansible_path,
+                                   "tor_v3_keys.json")
+
+    with open(secret_key_path) as f:
+        v3_onion_service_keys = json.load(f)
+
+    expected_keys = ['app_journalist_public_key',
+                     'app_journalist_private_key',
+                     'app_ssh_public_key',
+                     'app_ssh_private_key',
+                     'mon_ssh_public_key',
+                     'mon_ssh_private_key']
+    for key in expected_keys:
+        assert key in v3_onion_service_keys.keys()
+
+
+def test_find_or_generate_new_torv3_keys_subsequent_run(tmpdir, capsys):
+    args = argparse.Namespace(ansible_path=str(tmpdir))
+
+    secret_key_path = os.path.join(args.ansible_path,
+                                   "tor_v3_keys.json")
+    old_keys = {'foo': 'bar'}
+    with open(secret_key_path, 'w') as f:
+        json.dump(old_keys, f)
+
+    return_code = securedrop_admin.find_or_generate_new_torv3_keys(args)
+
+    captured = capsys.readouterr()
+    assert 'Tor v3 onion service keys already exist' in captured.out
+    assert return_code == 0
+
+    with open(secret_key_path) as f:
+        v3_onion_service_keys = json.load(f)
+
+    assert v3_onion_service_keys == old_keys
diff --git a/molecule/testinfra/staging/app/test_tor_hidden_services.py b/molecule/testinfra/staging/app/test_tor_hidden_services.py
--- a/molecule/testinfra/staging/app/test_tor_hidden_services.py
+++ b/molecule/testinfra/staging/app/test_tor_hidden_services.py
@@ -29,6 +29,7 @@ def test_tor_service_hostnames(host, tor_service):
     # Declare regex only for THS; we'll build regex for ATHS only if
     # necessary, since we won't have the required values otherwise.
     ths_hostname_regex = r"[a-z0-9]{16}\.onion"
+    ths_hostname_regex_v3 = r"[a-z0-9]{56}\.onion"
 
     with host.sudo():
         f = host.file("/var/lib/tor/services/{}/hostname".format(
@@ -41,15 +42,24 @@ def test_tor_service_hostnames(host, tor_service):
         # All hostnames should contain at *least* the hostname.
         assert re.search(ths_hostname_regex, f.content_string)
 
-        if tor_service['authenticated']:
+        if tor_service['authenticated'] and tor_service['version'] == 2:
             # HidServAuth regex is approximately [a-zA-Z0-9/+], but validating
             # the entire entry is sane, and we don't need to nitpick the
             # charset.
             aths_hostname_regex = ths_hostname_regex + " .{22} # client: " + \
                                   tor_service['client']
             assert re.search("^{}$".format(aths_hostname_regex), f.content_string)
-        else:
+        elif tor_service['authenticated'] and tor_service['version'] == 3:
+            # For authenticated version 3 onion services, the authorized_client
+            # directory will exist and contain a file called client.auth.
+            client_auth = host.file(
+                "/var/lib/tor/services/{}/authorized_clients/client.auth".format(
+                    tor_service['name']))
+            assert client_auth.is_file
+        elif tor_service['version'] == 2:
             assert re.search("^{}$".format(ths_hostname_regex), f.content_string)
+        else:
+            assert re.search("^{}$".format(ths_hostname_regex_v3), f.content_string)
 
 
 @pytest.mark.parametrize('tor_service', sdvars.tor_services)
@@ -61,7 +71,7 @@ def test_tor_services_config(host, tor_service):
       * HiddenServiceDir
       * HiddenServicePort
 
-    Only authenticated Onion Services must also include:
+    Only v2 authenticated Onion Services must also include:
 
       * HiddenServiceAuthorizeClient
 
@@ -81,7 +91,10 @@ def test_tor_services_config(host, tor_service):
 
     # Ensure that service is hardcoded to v2, for compatibility
     # with newer versions of Tor, which default to v3.
-    version_string = "HiddenServiceVersion 2"
+    if tor_service['version'] == 2:
+        version_string = "HiddenServiceVersion 2"
+    else:
+        version_string = ""
 
     port_regex = "HiddenServicePort {} 127.0.0.1:{}".format(
             remote_port, local_port)
@@ -89,9 +102,12 @@ def test_tor_services_config(host, tor_service):
     assert f.contains("^{}$".format(dir_regex))
     assert f.contains("^{}$".format(port_regex))
 
-    service_regex = "\n".join([dir_regex, version_string, port_regex])
+    if version_string:
+        service_regex = "\n".join([dir_regex, version_string, port_regex])
+    else:
+        service_regex = "\n".join([dir_regex, port_regex])
 
-    if tor_service['authenticated']:
+    if tor_service['authenticated'] and tor_service['version'] == 2:
         auth_regex = "HiddenServiceAuthorizeClient stealth {}".format(
                 tor_service['client'])
         assert f.contains("^{}$".format(auth_regex))