Repository: fedora-infra/bodhi

Issue:
Composing flatpaks does not work because they use a different repo name in Fedora
@owtaylor and I tested a flatpak compose today in Bodhi's staging, and found that it fails to find the flatpak in the registry because it expects it to have the Fedora version prepended into the repository name:

```
[2018-10-09 16:31:31][bodhi.server   ERROR]f29-flatpak-updates-testing /usr/bin/bodhi-skopeo-lite copy docker://candidate-regi[10/215]g 
.stg.fedoraproject.org/f29/flatpak-runtime:f29-20181008180041.2 returned a non-0 exit code: 1g 
[2018-10-09 16:31:31][bodhi.server   ERROR]f29-flatpak-updates-testingg
Traceback (most recent call last):g 
  File "/usr/bin/bodhi-skopeo-lite", line 11, in <module>g 
    load_entry_point('bodhi-server==3.10.0', 'console_scripts', 'bodhi-skopeo-lite')() 
  File "/usr/lib/python2.7/site-packages/click/core.py", line 722, in __call__g                     
    return self.main(*args, **kwargs)g 
  File "/usr/lib/python2.7/site-packages/click/core.py", line 697, in maing 
    rv = self.invoke(ctx) 
  File "/usr/lib/python2.7/site-packages/click/core.py", line 1066, in invokeg 
    return _process_result(sub_ctx.command.invoke(sub_ctx))g  
  File "/usr/lib/python2.7/site-packages/click/core.py", line 895, in invokeg 
    return ctx.invoke(self.callback, **ctx.params)g 
  File "/usr/lib/python2.7/site-packages/click/core.py", line 535, in invokeg 
    return callback(*args, **kwargs)g 
  File "/usr/lib/python2.7/site-packages/bodhi/server/scripts/skopeo_lite.py", line 768, in copyg 
    Copier(src.get_endpoint(), tmp).copy()g 
  File "/usr/lib/python2.7/site-packages/bodhi/server/scripts/skopeo_lite.py", line 726, in copyg 
    info = self.src.get_manifest()g 
  File "/usr/lib/python2.7/site-packages/bodhi/server/scripts/skopeo_lite.py", line 633, in get_manifestg 
    return get_manifest(self.session, self.repo, self.tag)g 
  File "/usr/lib/python2.7/site-packages/bodhi/server/scripts/skopeo_lite.py", line 345, in get_manifestg 
    response.raise_for_status()g 
  File "/usr/lib/python2.7/site-packages/requests/models.py", line 935, in raise_for_statusg 
    raise HTTPError(http_error_msg, response=self)g 
requests.exceptions.HTTPError: 404 Client Error: Not Found for url: https://candidate-registry.stg.fedoraproject.org/v2/f29/flatpak-rug 
ntime/manifests/f29-20181008180041.2g
```

The URL should not have had that ```f29/``` added before ```flatpak-runtime```, but that is hardcoded into Bodhi since containers have it.

@owtaylor suggested that we could use the Koji metadata to know the name of the registry repository instead of hardcoding it. For example, https://koji.stg.fedoraproject.org/koji/buildinfo?buildID=1137170 tells us that ```candidate-registry.stg.fedoraproject.org/flatpak-runtime:f29-20181008180041.2``` is a pull URL. This works for containers too:

```
koji call getBuild cockpit-176-5.fc28
{'build_id': 1141835,
 'completion_time': '2018-09-04 13:38:12',
 'completion_ts': 1536068292.0,
 'creation_event_id': 36252316,
 'creation_time': '2018-09-04 13:38:15.365648',
 'creation_ts': 1536068295.36565,
 'epoch': None,
 'extra': {'container_koji_task_id': 29485206,
           'image': {'autorebuild': False,
                     'help': None,
                     'index': {'digests': {'application/vnd.docker.distribution.manifest.list.v2+json': 'sha256:2bd64a888535a6b0428a3cfcaa4aa7cec5dd3e3bbb4ed84e315e985e2d5c6bb5'},
                               'pull': ['candidate-registry.fedoraproject.org/f29/cockpit@sha256:2bd64a888535a6b0428a3cfcaa4aa7cec5dd3e3bbb4ed84e315e985e2d5c6bb5',
                                        'candidate-registry.fedoraproject.org/f29/cockpit:176-5.fc28'],
                               'tags': ['latest', '176', '176-5.fc28']},
                     'isolated': False,
                     'parent_image_builds': {'registry.fedoraproject.org/fedora:29': None}},
           'submitter': 'containerbuild'},
 'id': 1141835,
 'name': 'cockpit',
 'nvr': 'cockpit-176-5.fc28',
 'owner_id': 3998,
 'owner_name': 'cverna',
 'package_id': 17959,
 'package_name': 'cockpit',
 'release': '5.fc28',
 'source': 'https://src.fedoraproject.org/container/cockpit.git#851196158dc74a4ba970c1024bde58be86d49f25',
 'start_time': '2018-09-04 13:17:46',
 'start_ts': 1536067066.0,
 'state': 1,
 'task_id': None,
 'version': '176',
 'volume_id': 0,
 'volume_name': 'DEFAULT'}
```


Discussion:

Resolving patch:
diff --git a/bodhi/server/buildsys.py b/bodhi/server/buildsys.py
--- a/bodhi/server/buildsys.py
+++ b/bodhi/server/buildsys.py
@@ -253,22 +253,41 @@ def getBuild(self, build='TurboGears-1.0.2.2-2.fc17', other=False, testing=False
         for token in release_tokens:
             # Starting to hardcode some dev buildsys bits for docker.
             # See https://github.com/fedora-infra/bodhi/pull/1543
-            if token.endswith("container"):
-                tag = "f%s-updates-testing" % token.replace("fc", "").replace("container", "")
-                data['extra'] = {
-                    'container_koji_task_id': 19708268,
-                    'image': {},
+            if token.endswith("container") or token.endswith("flatpak"):
+                fedora_release = "f" + (token
+                                        .replace("fc", "")
+                                        .replace("flatpak", "")
+                                        .replace("container", ""))
+                tag = "%s-updates-testing" % fedora_release
+
+                format_data = {
+                    'registry': 'candidate-registry.fedoraproject.org',
+                    'hash': 'sha256:2bd64a888...',
+                    'version': version,
+                    'release': release
                 }
-                break
 
-            if token.endswith("flatpak"):
-                tag = "f%s-updates-testing" % token.replace("fc", "")
+                if token.endswith("flatpak"):
+                    format_data['repository'] = name
+                else:
+                    tag = "f%s-updates-testing" % token.replace("fc", "").replace("container", "")
+                    format_data['repository'] = "{}/{}".format(fedora_release, name)
+
                 data['extra'] = {
-                    'container_koji_task_id': 21098765,
+                    'container_koji_task_id': 19708268,
                     'image': {
-                        'flatpak': True
+                        'index': {
+                            'pull': ['{registry}/{repository}:{hash}'
+                                     .format(**format_data),
+                                     '{registry}/{repository}:{version}:{release}'
+                                     .format(**format_data)],
+                        }
                     },
                 }
+
+                if token.endswith("flatpak"):
+                    data['extra']['image']['flatpak'] = True
+
                 break
 
             # Hardcoding for modules in the dev buildsys
diff --git a/bodhi/server/util.py b/bodhi/server/util.py
--- a/bodhi/server/util.py
+++ b/bodhi/server/util.py
@@ -1426,6 +1426,31 @@ def __exit__(self, *args, **kwargs):
         self.session.autoflush = self.autoflush
 
 
+def _get_build_repository(build):
+    """
+    Return the registry repository name for the given Build.
+
+    For example, if the Build's container pull string is:
+
+    'candidate-registry.fedoraproject.org/f29/cockpit:176-5.fc28', this will return 'f29/cockpit'.
+
+    Args:
+        build (bodhi.server.models.Build): A Build representing a container or flatpak.
+    Returns:
+        str: The registry repository name for the build.
+    """
+    koji = buildsys.get_session()
+    koji_build = koji.getBuild(build.nvr)
+
+    pull_specs = koji_build['extra']['image']['index']['pull']
+    # Specs are of the form 'candidate-registry.fedoraproject.org/f29/cockpit:176-5.fc28'
+    # All the pull specs should have the same repository, so which one we use is arbitrary
+    base, tag = pull_specs[0].split(':', 1)
+    server, repository = base.split('/', 1)
+
+    return repository
+
+
 def copy_container(build, destination_registry=None, destination_tag=None):
     """
     Copy a ContainerBuild from the source registry to a destination registry under the given tag.
@@ -1442,12 +1467,17 @@ def copy_container(build, destination_registry=None, destination_tag=None):
         RuntimeError: If skopeo returns a non-0 exit code.
     """
     source_registry = config['container.source_registry']
+    source_tag = '{}-{}'.format(build.nvr_version, build.nvr_release)
 
+    if destination_tag is None:
+        destination_tag = source_tag
     if destination_registry is None:
         destination_registry = config['container.destination_registry']
 
-    source_url = _container_image_url(build, source_registry)
-    destination_url = _container_image_url(build, destination_registry, destination_tag)
+    repository = _get_build_repository(build)
+
+    source_url = _container_image_url(source_registry, repository, source_tag)
+    destination_url = _container_image_url(destination_registry, repository, destination_tag)
 
     skopeo_cmd = [
         config.get('skopeo.cmd'), 'copy', source_url, destination_url]
@@ -1457,7 +1487,7 @@ def copy_container(build, destination_registry=None, destination_tag=None):
     cmd(skopeo_cmd, raise_on_error=True)
 
 
-def _container_image_url(build, registry, tag=None):
+def _container_image_url(registry, repository, tag=None):
     """
     Return a URL suitable for use in Skopeo for copying or deleting container images.
 
@@ -1466,19 +1496,13 @@ def _container_image_url(build, registry, tag=None):
     docker://registry.fedoraproject.org/f27/httpd:build is tag is 'build'.
 
     Args:
-        build (bodhi.server.models.ContainerBuild): The build you want a URL to reference.
+        repository (str): the repository name in the registry, extracted from Koji metadata
         registry (str): The registry you want a URL for.
-        tag (str or None): If given, the tag in the registry you want to reference. If None (the
-            default), the build's version and release will be used to form a tag.
+        tag (str or None): The tag in the registry you want to reference.
     Returns:
         str: A URL referencing the given build and tag in the given registry.
     """
-    image_name = '{}/{}'.format(build.release.branch, build.package.name)
-
-    if tag is None:
-        tag = '{}-{}'.format(build.nvr_version, build.nvr_release)
-
-    return 'docker://{}/{}:{}'.format(registry, image_name, tag)
+    return 'docker://{}/{}:{}'.format(registry, repository, tag)
 
 
 def get_absolute_path(location):


Test patch:
diff --git a/bodhi/tests/server/consumers/test_masher.py b/bodhi/tests/server/consumers/test_masher.py
--- a/bodhi/tests/server/consumers/test_masher.py
+++ b/bodhi/tests/server/consumers/test_masher.py
@@ -2294,14 +2294,16 @@ def setUp(self):
         package2 = Package(name=u'testcontainer2',
                            type=ContentType.container)
         self.db.add(package2)
-        build1 = ContainerBuild(nvr=u'testcontainer1-2.0.1-71.fc28', release=release, signed=True,
+        build1 = ContainerBuild(nvr=u'testcontainer1-2.0.1-71.fc28container',
+                                release=release, signed=True,
                                 package=package1)
         self.db.add(build1)
-        build2 = ContainerBuild(nvr=u'testcontainer2-1.0.1-1.fc28', release=release, signed=True,
+        build2 = ContainerBuild(nvr=u'testcontainer2-1.0.1-1.fc28container',
+                                release=release, signed=True,
                                 package=package2)
         self.db.add(build2)
         update = Update(
-            title=u'testcontainer1-2.0.1-71.fc28, testcontainer2-1.0.1-1.fc28',
+            title=u'testcontainer1-2.0.1-71.fc28container, testcontainer2-1.0.1-1.fc28container',
             builds=[build1, build2], user=user,
             status=UpdateStatus.pending,
             request=UpdateRequest.testing,
@@ -2328,7 +2330,8 @@ def test_request_not_stable(self, Popen):
         # Popen should have been called three times per build, once for each of the destination
         # tags. With two builds that is a total of 6 calls to Popen.
         expected_mock_calls = []
-        for source in ('testcontainer1:2.0.1-71.fc28', 'testcontainer2:1.0.1-1.fc28'):
+        for source in ('testcontainer1:2.0.1-71.fc28container',
+                       'testcontainer2:1.0.1-1.fc28container'):
             for dtag in [source.split(':')[1], source.split(':')[1].split('-')[0], 'testing']:
                 mock_call = mock.call(
                     [config['skopeo.cmd'], 'copy',
@@ -2356,7 +2359,8 @@ def test_request_stable(self, Popen):
         # Popen should have been called three times per build, once for each of the destination
         # tags. With two builds that is a total of 6 calls to Popen.
         expected_mock_calls = []
-        for source in ('testcontainer1:2.0.1-71.fc28', 'testcontainer2:1.0.1-1.fc28'):
+        for source in ('testcontainer1:2.0.1-71.fc28container',
+                       'testcontainer2:1.0.1-1.fc28container'):
             for dtag in [source.split(':')[1], source.split(':')[1].split('-')[0], 'latest']:
                 mock_call = mock.call(
                     [config['skopeo.cmd'], 'copy',
@@ -2385,9 +2389,9 @@ def test_skopeo_error_code(self, Popen):
         # Popen should have been called once.
         skopeo_cmd = [
             config['skopeo.cmd'], 'copy',
-            'docker://{}/f28/testcontainer1:2.0.1-71.fc28'.format(
+            'docker://{}/f28/testcontainer1:2.0.1-71.fc28container'.format(
                 config['container.source_registry']),
-            'docker://{}/f28/testcontainer1:2.0.1-71.fc28'.format(
+            'docker://{}/f28/testcontainer1:2.0.1-71.fc28container'.format(
                 config['container.destination_registry'])]
         Popen.assert_called_once_with(skopeo_cmd, shell=False, stderr=-1, stdout=-1, cwd=None)
         self.assertEqual(str(exc.exception),
@@ -2409,7 +2413,8 @@ def test_skopeo_extra_copy_flags(self, Popen):
         # Popen should have been called three times per build, once for each of the destination
         # tags. With two builds that is a total of 6 calls to Popen.
         expected_mock_calls = []
-        for source in ('testcontainer1:2.0.1-71.fc28', 'testcontainer2:1.0.1-1.fc28'):
+        for source in ('testcontainer1:2.0.1-71.fc28container',
+                       'testcontainer2:1.0.1-1.fc28container'):
             for dtag in [source.split(':')[1], source.split(':')[1].split('-')[0], 'testing']:
                 mock_call = mock.call(
                     [config['skopeo.cmd'], 'copy', '--dest-tls-verify=false',
@@ -2455,14 +2460,16 @@ def setUp(self):
         package2 = Package(name=u'testflatpak2',
                            type=ContentType.flatpak)
         self.db.add(package2)
-        build1 = FlatpakBuild(nvr=u'testflatpak1-2.0.1-71.fc28', release=release, signed=True,
+        build1 = FlatpakBuild(nvr=u'testflatpak1-2.0.1-71.fc28flatpak',
+                              release=release, signed=True,
                               package=package1)
         self.db.add(build1)
-        build2 = FlatpakBuild(nvr=u'testflatpak2-1.0.1-1.fc28', release=release, signed=True,
+        build2 = FlatpakBuild(nvr=u'testflatpak2-1.0.1-1.fc28flatpak',
+                              release=release, signed=True,
                               package=package2)
         self.db.add(build2)
         update = Update(
-            title=u'testflatpak1-2.0.1-71.fc28, testflatpak2-1.0.1-1.fc28',
+            title=u'testflatpak1-2.0.1-71.fc28flatpak, testflatpak2-1.0.1-1.fc28flatpak',
             builds=[build1, build2], user=user,
             status=UpdateStatus.pending,
             request=UpdateRequest.testing,
@@ -2494,13 +2501,13 @@ def test_flatpak_compose(self, Popen):
         # Popen should have been called three times per build, once for each of the destination
         # tags. With two builds that is a total of 6 calls to Popen.
         expected_mock_calls = []
-        for source in ('testflatpak1:2.0.1-71.fc28', 'testflatpak2:1.0.1-1.fc28'):
+        for source in ('testflatpak1:2.0.1-71.fc28flatpak', 'testflatpak2:1.0.1-1.fc28flatpak'):
             for dtag in [source.split(':')[1], source.split(':')[1].split('-')[0], 'testing']:
                 mock_call = mock.call(
                     [config['skopeo.cmd'], 'copy',
-                     'docker://{}/f28/{}'.format(config['container.source_registry'], source),
-                     'docker://{}/f28/{}:{}'.format(config['container.destination_registry'],
-                                                    source.split(':')[0], dtag)],
+                     'docker://{}/{}'.format(config['container.source_registry'], source),
+                     'docker://{}/{}:{}'.format(config['container.destination_registry'],
+                                                source.split(':')[0], dtag)],
                     shell=False, stderr=-1, stdout=-1, cwd=None)
                 expected_mock_calls.append(mock_call)
                 expected_mock_calls.append(mock.call().communicate())