Repository: freedomofpress/securedrop

Issue:
Port upgrade scenario to Xenial
## Description

The "upgrade" scenario in Molecule is Trusty-only, and should be ported to Xenial (as default), so that upgrade testing can be performed on Xenial hosts. Related to #4155.

## User Research Evidence
None, dev-env only. 

## User Stories

As a developer, I want to test upgrades on the primary supported platform, i.e. Xenial. 


Discussion:

Resolving patch:
diff --git a/molecule/vagrant-packager-trusty/package.py b/molecule/vagrant-packager-trusty/package.py
new file mode 120000
--- /dev/null
+++ b/molecule/vagrant-packager-trusty/package.py
@@ -0,0 +1 @@
+../vagrant-packager/package.py
\ No newline at end of file
diff --git a/molecule/vagrant_packager/package.py b/molecule/vagrant-packager/package.py
similarity index 65%
rename from molecule/vagrant_packager/package.py
rename to molecule/vagrant-packager/package.py
--- a/molecule/vagrant_packager/package.py
+++ b/molecule/vagrant-packager/package.py
@@ -3,6 +3,7 @@
 #
 #
 #
+import hashlib
 import json
 import os
 from os.path import join
@@ -13,6 +14,17 @@
 import xml.etree.ElementTree as ET
 
 
+# Current script is symlinked into adjacent scenario, for Trusty compatibility.
+# Look up "name" for scenario from real path (relative to symlink), but store
+# all artifacts in primary scenario (via realpath).
+SCENARIO_NAME = os.path.basename(os.path.dirname(os.path.abspath(__file__)))
+SCENARIO_PATH = os.path.dirname(os.path.realpath(__file__))
+BOX_BUILD_DIR = join(SCENARIO_PATH, "build")
+BOX_METADATA_DIR = join(SCENARIO_PATH, "box_files")
+EPHEMERAL_DIRS = {}
+TARGET_VERSION_FILE = os.path.join(SCENARIO_PATH, os.path.pardir, "shared", "stable.ver")
+
+
 class LibVirtPackager(object):
 
     def __init__(self, vm):
@@ -117,17 +129,14 @@ def vagrant_metadata(self, img_location):
 
 
 def main():
-    SCENARIO_PATH = os.path.dirname(os.path.realpath(__file__))
-    BOX_PATH = join(SCENARIO_PATH, "build")
-    EPHEMERAL_DIRS = {}
-    TARGET_VERSION_FILE = os.path.join(SCENARIO_PATH, os.path.pardir, "shared", "stable.ver")
     with open(TARGET_VERSION_FILE, 'r') as f:
         TARGET_VERSION = f.read().strip()
-    try:
-        TARGET_PLATFORM = os.environ['SECUREDROP_TARGET_PLATFORM']
-    except KeyError:
-        msg = "Set SECUREDROP_TARGET_PLATFORM env var to 'trusty' or 'xenial'"
-        raise Exception(msg)
+
+    # Default to Xenial as base OS, but detect if script was invoked from the
+    # Trusty-specific scenario, and use Trusty if so.
+    TARGET_PLATFORM = os.environ.get("SECUREDROP_TARGET_PLATFORM", "xenial")
+    if SCENARIO_NAME.endswith("-trusty"):
+        TARGET_PLATFORM = "trusty"
 
     for srv in ["app-staging", "mon-staging"]:
 
@@ -141,7 +150,7 @@ def main():
             except OSError:
                 pass
 
-        vm = LibVirtPackager("vagrant_packager_"+srv)
+        vm = LibVirtPackager("{}_{}".format(SCENARIO_NAME, srv))
 
         tmp_img_file = join(EPHEMERAL_DIRS["tmp"], "wip.img")
         packaged_img_file = join(EPHEMERAL_DIRS["build"], "box.img")
@@ -178,11 +187,11 @@ def main():
                 mdata)
 
         # Copy in appropriate vagrant file to build dir
-        shutil.copyfile(join(SCENARIO_PATH, "box_files", "Vagrantfile."+srv),
+        shutil.copyfile(join(BOX_METADATA_DIR, "Vagrantfile."+srv),
                         join(EPHEMERAL_DIRS['build'], 'Vagrantfile'))
 
         print("Creating tar file")
-        box_file = join(BOX_PATH, "{}-{}_{}.box".format(srv, TARGET_PLATFORM, TARGET_VERSION))
+        box_file = join(BOX_BUILD_DIR, "{}-{}_{}.box".format(srv, TARGET_PLATFORM, TARGET_VERSION))
         with tarfile.open(box_file, "w|gz") as tar:
             for boxfile in ["box.img", "Vagrantfile", "metadata.json"]:
                 tar.add(join(EPHEMERAL_DIRS["build"], boxfile),
@@ -190,9 +199,68 @@ def main():
 
         print("Box created at {}".format(box_file))
 
+        print("Updating box metadata")
+        update_box_metadata(srv, box_file, TARGET_PLATFORM, TARGET_VERSION)
+
         print("Clean-up tmp space")
         shutil.rmtree(EPHEMERAL_DIRS['tmp'])
 
 
+def sha256_checksum(filepath):
+    """
+    Returns a SHA256 checksum for a given filepath.
+    """
+    checksum = hashlib.sha256()
+    with open(filepath, 'rb') as f:
+        # Read by chunks, to avoid slurping the entire file into memory.
+        # Box files range from 500MB to 1.5GB.
+        for block in iter(lambda: f.read(checksum.block_size), b''):
+            checksum.update(block)
+    return checksum.hexdigest()
+
+
+def update_box_metadata(server_name, box_file, platform, version):
+    """
+    Updates the JSON file of Vagrant box metadata, including remote URL,
+    version number, and SHA256 checksum.
+    """
+    # Strip off "staging" suffix from box names
+    server_name_short = re.sub('\-staging$', '', server_name)
+    json_file_basename = "{}_{}_metadata.json".format(server_name_short, platform)
+    json_file = os.path.join(BOX_METADATA_DIR, json_file_basename)
+
+    # Read in current JSON metadata, so we can append the new info to it.
+    with open(json_file, "r") as f:
+        metadata_config = json.loads(f.read())
+
+    base_url = "https://s3.amazonaws.com/securedrop-vagrant"
+    box_name = os.path.basename(box_file)
+    box_url = "{}/{}".format(base_url, box_name)
+    box_checksum = sha256_checksum(box_file)
+    box_config = dict(
+        name="libvirt",
+        url=box_url,
+        checksum_type="sha256",
+        checksum=box_checksum,
+    )
+    # Creating list of dicts to adhere to JSON format of Vagrant box metadata
+    providers_list = []
+    providers_list.append(box_config)
+    version_config = dict(
+        version=version,
+        providers=providers_list,
+    )
+    box_versions = metadata_config['versions']
+    box_versions.append(version_config)
+    metadata_config['versions'] = box_versions
+
+    # Write out final, modified data. Does not validate for uniqueness,
+    # so repeated runs on the same version will duplicate version info,
+    # which'll likely break the box fetching. Target file is version-controlled,
+    # though, so easy enough to correct in the event of a mistake.
+    with open(json_file, "w") as f:
+        f.write(json.dumps(metadata_config, indent=2, sort_keys=True))
+
+
 if __name__ == "__main__":
     main()


Test patch:
diff --git a/docs/development/upgrade_testing.rst b/docs/development/upgrade_testing.rst
--- a/docs/development/upgrade_testing.rst
+++ b/docs/development/upgrade_testing.rst
@@ -29,6 +29,12 @@ https://apt-test.freedom.press/. Both options are described below.
 Upgrade testing using locally-built packages
 --------------------------------------------
 
+.. note::
+   As of ``0.12.1``, the default platform for upgrade testing
+   boxes is Ubuntu Xenial 16.04. For a limited time, we will also support
+   upgrade boxes based on Ubuntu Trusty 14.04. Substitute ``upgrade-trusty``
+   for ``upgrade`` in the actions documented below to force use of Trusty.
+
 First, build the app code packages and create the environment:
 
 .. code:: sh
@@ -120,13 +126,8 @@ testing. The procedure is as follows:
 
 1. ``git checkout <version>``
 2. ``make vagrant-package``
-3. ``mv molecule/vagrant_packager/build/app-staging{,_<version>}.box``
-4. ``mv molecule/vagrant_packager/build/mon-staging{,_<version>}.box``
-5. ``sha256sum molecule/vagrant_packager/build/*.box``
-6. Manually update ``molecule/vagrant_packager/box_files/*.json`` with new
-   version information, including URL and checksum.
-7. ``cd molecule/vagrant_packager && ./push.yml`` to upload to S3
-8. Commit the local changes to JSON files and open a PR.
+3. ``cd molecule/vagrant-packager && ./push.yml`` to upload to S3
+4. Commit the local changes to JSON files and open a PR.
 
 Subsequent invocations of ``make upgrade-start`` will pull the latest
 version of the box.