Repository: fedora-infra/bodhi

Issue:
Add support for containers to the masher
We need the masher to be able to promote containers into testing and stable. We will probably use skopeo for this.

Depends on #2027.


Discussion:
@runcom told me that @mtrmac is working on the support for manifest lists in skopeo (CC @owtaylor) #you-are-now-connected
On 01/29/2018 04:21 PM, Tomas Tomecek wrote:
> @runcom <https://github.com/runcom> told me that @mtrmac
> <https://github.com/mtrmac> is working on the support for manifest lists
> in skopeo (CC @owtaylor <https://github.com/owtaylor>)
> #you-are-now-connected

#so-connected

@nalind is actually working on that in containers/image (the library backing `skopeo`), in https://github.com/containers/image/pull/400 .

Resolving patch:
diff --git a/bodhi/server/config.py b/bodhi/server/config.py
--- a/bodhi/server/config.py
+++ b/bodhi/server/config.py
@@ -359,6 +359,12 @@ class BodhiConfig(dict):
         'captcha.ttl': {
             'value': 300,
             'validator': int},
+        'container.destination_registry': {
+            'value': 'registry.fedoraproject.org',
+            'validator': six.text_type},
+        'container.source_registry': {
+            'value': 'candidate-registry.fedoraproject.org',
+            'validator': six.text_type},
         'cors_connect_src': {
             'value': 'https://*.fedoraproject.org/ wss://hub.fedoraproject.org:9939/',
             'validator': six.text_type},
@@ -548,6 +554,14 @@ class BodhiConfig(dict):
         'site_requirements': {
             'value': 'dist.rpmdeplint dist.upgradepath',
             'validator': six.text_type},
+        'skopeo.cmd': {
+            'value': '/usr/bin/skopeo',
+            'validator': six.text_type,
+        },
+        'skopeo.extra_copy_flags': {
+            'value': '',
+            'validator': six.text_type,
+        },
         'smtp_server': {
             'value': None,
             'validator': _validate_none_or(six.text_type)},
diff --git a/bodhi/server/consumers/masher.py b/bodhi/server/consumers/masher.py
--- a/bodhi/server/consumers/masher.py
+++ b/bodhi/server/consumers/masher.py
@@ -48,7 +48,8 @@
 from bodhi.server.metadata import UpdateInfoMetadata
 from bodhi.server.models import (Compose, ComposeState, Update, UpdateRequest, UpdateType, Release,
                                  UpdateStatus, ReleaseState, ContentType)
-from bodhi.server.util import sorted_updates, sanity_check_repodata, transactional_session_maker
+from bodhi.server.util import (get_nvr, sorted_updates, sanity_check_repodata,
+                               transactional_session_maker)
 
 
 def checkpoint(method):
@@ -305,7 +306,7 @@ def get_masher(content_type):
         ComposerThread or None: Either a ContainerComposerThread, RPMComposerThread, or a
             ModuleComposerThread, as appropriate, or None if no masher is found.
     """
-    mashers = [RPMComposerThread, ModuleComposerThread]
+    mashers = [ContainerComposerThread, RPMComposerThread, ModuleComposerThread]
     for possible in mashers:
         if possible.ctype is content_type:
             return possible
@@ -860,6 +861,49 @@ def sort_by_days_in_testing(self, updates):
         return updates
 
 
+class ContainerComposerThread(ComposerThread):
+    """Use skopeo to copy and tag container images."""
+
+    ctype = ContentType.container
+
+    def _compose_updates(self):
+        """Use skopeo to copy images to the correct repos and tags."""
+        source_registry = config['container.source_registry']
+        destination_registry = config['container.destination_registry']
+
+        for update in self.compose.updates:
+
+            if update.request is UpdateRequest.stable:
+                destination_tag = 'latest'
+            else:
+                destination_tag = 'testing'
+
+            for build in update.builds:
+                image_name = '{}/{}'.format(build.release.branch, build.package.name)
+                name, version, release = get_nvr(build.nvr)
+                version_release = '{}-{}'.format(version, release)
+                source_url = 'docker://{}/{}:{}'.format(source_registry, image_name,
+                                                        version_release)
+                for dtag in [version_release, version, destination_tag]:
+                    destination_url = 'docker://{}/{}:{}'.format(destination_registry, image_name,
+                                                                 dtag)
+                    skopeo_cmd = [
+                        config.get('skopeo.cmd'), 'copy', source_url, destination_url]
+                    if config.get('skopeo.extra_copy_flags'):
+                        skopeo_cmd.insert(2, config.get('skopeo.extra_copy_flags'))
+                    log.debug('Running {}'.format(' '.join(skopeo_cmd)))
+                    skopeo_process = subprocess.Popen(
+                        skopeo_cmd, shell=False, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
+                    out, err = skopeo_process.communicate()
+
+                    if skopeo_process.returncode:
+                        msg = '{} returned a non-0 exit code: {}'.format(
+                            ' '.join(skopeo_cmd), skopeo_process.returncode)
+                        log.error(msg)
+                        log.error(out + err)
+                        raise RuntimeError(msg)
+
+
 class PungiComposerThread(ComposerThread):
     """Compose update with Pungi."""
 


Test patch:
diff --git a/bodhi/tests/server/consumers/test_masher.py b/bodhi/tests/server/consumers/test_masher.py
--- a/bodhi/tests/server/consumers/test_masher.py
+++ b/bodhi/tests/server/consumers/test_masher.py
@@ -35,12 +35,13 @@
 
 from bodhi.server import buildsys, exceptions, log, push
 from bodhi.server.config import config
-from bodhi.server.consumers.masher import (checkpoint, Masher, ComposerThread, RPMComposerThread,
-                                           ModuleComposerThread, PungiComposerThread)
+from bodhi.server.consumers.masher import (
+    checkpoint, Masher, ComposerThread, ContainerComposerThread, RPMComposerThread,
+    ModuleComposerThread, PungiComposerThread)
 from bodhi.server.exceptions import LockedUpdateException
 from bodhi.server.models import (
-    Build, BuildrootOverride, Compose, ComposeState, Release, ReleaseState, RpmBuild,
-    TestGatingStatus, Update, UpdateRequest, UpdateStatus, UpdateType, User, ModuleBuild,
+    Build, BuildrootOverride, Compose, ComposeState, ContainerBuild, Release, ReleaseState,
+    RpmBuild, TestGatingStatus, Update, UpdateRequest, UpdateStatus, UpdateType, User, ModuleBuild,
     ContentType, Package)
 from bodhi.server.util import mkmetadatadir
 from bodhi.tests.server import base
@@ -1758,10 +1759,8 @@ def test_expire_buildroot_overrides_exception(self, expire, exception_log, publi
 
 
 class ComposerThreadBaseTestCase(base.BaseTestCase):
-    """
-    This test class has common setUp() and tearDown() methods that are useful for testing the
-    ComposerThread class.
-    """
+    """Methods that are useful for testing ComposerThread subclasses."""
+
     def setUp(self):
         """
         Set up the test conditions.
@@ -1800,6 +1799,149 @@ def _make_msg(self, extra_push_args=None):
         return _make_msg(base.TransactionalSessionMaker(self.Session), extra_push_args)
 
 
+class TestContainerComposerThread__compose_updates(ComposerThreadBaseTestCase):
+    """Test ContainerComposerThread._compose_update()."""
+
+    def setUp(self):
+        super(TestContainerComposerThread__compose_updates, self).setUp()
+
+        user = self.db.query(User).first()
+        release = self.create_release('28C')
+        release.branch = 'f28'
+        package1 = Package(name=u'testcontainer1',
+                           type=ContentType.container)
+        self.db.add(package1)
+        package2 = Package(name=u'testcontainer2',
+                           type=ContentType.container)
+        self.db.add(package2)
+        build1 = ContainerBuild(nvr=u'testcontainer1-2.0.1-71.fc28', release=release, signed=True,
+                                package=package1)
+        self.db.add(build1)
+        build2 = ContainerBuild(nvr=u'testcontainer2-1.0.1-1.fc28', release=release, signed=True,
+                                package=package2)
+        self.db.add(build2)
+        update = Update(
+            title=u'testcontainer1-2.0.1-71.fc28, testcontainer2-1.0.1-1.fc28',
+            builds=[build1, build2], user=user,
+            status=UpdateStatus.pending,
+            request=UpdateRequest.testing,
+            notes=u'Neat I can compose containers now', release=release,
+            test_gating_status=TestGatingStatus.passed)
+        update.type = UpdateType.bugfix
+        self.db.add(update)
+        # Wipe out the tag cache so it picks up our new release
+        Release._tag_cache = None
+        self.db.flush()
+
+    @mock.patch('bodhi.server.consumers.masher.subprocess.Popen')
+    def test_request_not_stable(self, Popen):
+        """Ensure that the correct destination tag is used for non-stable updates."""
+        Popen.return_value.communicate.return_value = ('out', 'err')
+        Popen.return_value.returncode = 0
+        msg = self._make_msg(['--releases', 'F28C'])
+        t = ContainerComposerThread(self.semmock, msg['body']['msg']['composes'][0],
+                                    'bowlofeggs', log, self.Session, self.tempdir)
+        t.compose = Compose.from_dict(self.db, msg['body']['msg']['composes'][0])
+
+        t._compose_updates()
+
+        # Popen should have been called three times per build, once for each of the destination
+        # tags. With two builds that is a total of 6 calls to Popen.
+        expected_mock_calls = []
+        for source in ('testcontainer1:2.0.1-71.fc28', 'testcontainer2:1.0.1-1.fc28'):
+            for dtag in [source.split(':')[1], source.split(':')[1].split('-')[0], 'testing']:
+                mock_call = mock.call(
+                    [config['skopeo.cmd'], 'copy',
+                     'docker://{}/f28/{}'.format(config['container.source_registry'], source),
+                     'docker://{}/f28/{}:{}'.format(config['container.destination_registry'],
+                                                    source.split(':')[0], dtag)],
+                    shell=False, stderr=-1, stdout=-1)
+                expected_mock_calls.append(mock_call)
+                expected_mock_calls.append(mock.call().communicate())
+        self.assertEqual(Popen.mock_calls, expected_mock_calls)
+
+    @mock.patch('bodhi.server.consumers.masher.subprocess.Popen')
+    def test_request_stable(self, Popen):
+        """Ensure that the correct destination tag is used for stable updates."""
+        Popen.return_value.communicate.return_value = ('out', 'err')
+        Popen.return_value.returncode = 0
+        ContainerBuild.query.first().update.request = UpdateRequest.stable
+        msg = self._make_msg(['--releases', 'F28C'])
+        t = ContainerComposerThread(self.semmock, msg['body']['msg']['composes'][0],
+                                    'bowlofeggs', log, self.Session, self.tempdir)
+        t.compose = Compose.from_dict(self.db, msg['body']['msg']['composes'][0])
+
+        t._compose_updates()
+
+        # Popen should have been called three times per build, once for each of the destination
+        # tags. With two builds that is a total of 6 calls to Popen.
+        expected_mock_calls = []
+        for source in ('testcontainer1:2.0.1-71.fc28', 'testcontainer2:1.0.1-1.fc28'):
+            for dtag in [source.split(':')[1], source.split(':')[1].split('-')[0], 'latest']:
+                mock_call = mock.call(
+                    [config['skopeo.cmd'], 'copy',
+                     'docker://{}/f28/{}'.format(config['container.source_registry'], source),
+                     'docker://{}/f28/{}:{}'.format(config['container.destination_registry'],
+                                                    source.split(':')[0], dtag)],
+                    shell=False, stderr=-1, stdout=-1)
+                expected_mock_calls.append(mock_call)
+                expected_mock_calls.append(mock.call().communicate())
+        self.assertEqual(Popen.mock_calls, expected_mock_calls)
+
+    @mock.patch('bodhi.server.consumers.masher.subprocess.Popen')
+    def test_skopeo_error_code(self, Popen):
+        """Assert that a RuntimeError is raised if skopeo returns a non-0 exit code."""
+        Popen.return_value.communicate.return_value = ('out', 'err')
+        Popen.return_value.returncode = 1
+        ContainerBuild.query.first().update.request = UpdateRequest.stable
+        msg = self._make_msg(['--releases', 'F28C'])
+        t = ContainerComposerThread(self.semmock, msg['body']['msg']['composes'][0],
+                                    'bowlofeggs', log, self.Session, self.tempdir)
+        t.compose = Compose.from_dict(self.db, msg['body']['msg']['composes'][0])
+
+        with self.assertRaises(RuntimeError) as exc:
+            t._compose_updates()
+
+        # Popen should have been called once.
+        skopeo_cmd = [
+            config['skopeo.cmd'], 'copy',
+            'docker://{}/f28/testcontainer1:2.0.1-71.fc28'.format(
+                config['container.source_registry']),
+            'docker://{}/f28/testcontainer1:2.0.1-71.fc28'.format(
+                config['container.destination_registry'])]
+        Popen.assert_called_once_with(skopeo_cmd, shell=False, stderr=-1, stdout=-1)
+        self.assertEqual(str(exc.exception),
+                         '{} returned a non-0 exit code: 1'.format(' '.join(skopeo_cmd)))
+
+    @mock.patch.dict(config, {'skopeo.extra_copy_flags': '--dest-tls-verify=false'})
+    @mock.patch('bodhi.server.consumers.masher.subprocess.Popen')
+    def test_skopeo_extra_copy_flags(self, Popen):
+        """Test the skopeo.extra_copy_flags setting."""
+        Popen.return_value.communicate.return_value = ('out', 'err')
+        Popen.return_value.returncode = 0
+        msg = self._make_msg(['--releases', 'F28C'])
+        t = ContainerComposerThread(self.semmock, msg['body']['msg']['composes'][0],
+                                    'bowlofeggs', log, self.Session, self.tempdir)
+        t.compose = Compose.from_dict(self.db, msg['body']['msg']['composes'][0])
+
+        t._compose_updates()
+
+        # Popen should have been called three times per build, once for each of the destination
+        # tags. With two builds that is a total of 6 calls to Popen.
+        expected_mock_calls = []
+        for source in ('testcontainer1:2.0.1-71.fc28', 'testcontainer2:1.0.1-1.fc28'):
+            for dtag in [source.split(':')[1], source.split(':')[1].split('-')[0], 'testing']:
+                mock_call = mock.call(
+                    [config['skopeo.cmd'], 'copy', '--dest-tls-verify=false',
+                     'docker://{}/f28/{}'.format(config['container.source_registry'], source),
+                     'docker://{}/f28/{}:{}'.format(config['container.destination_registry'],
+                                                    source.split(':')[0], dtag)],
+                    shell=False, stderr=-1, stdout=-1)
+                expected_mock_calls.append(mock_call)
+                expected_mock_calls.append(mock.call().communicate())
+        self.assertEqual(Popen.mock_calls, expected_mock_calls)
+
+
 class TestPungiComposerThread__get_master_repomd_url(ComposerThreadBaseTestCase):
     """This class contains tests for the PungiComposerThread._get_master_repomd_url() method."""
     @mock.patch.dict(