Repository: fedora-infra/bodhi

Issue:
Bodhi polls wrong master repo URL for alternative arches during mash
Last night's Fedora 26 mash failed with this traceback:

```
[2017-03-10 09:54:58][    fedmsg    INFO]f26-updates-testing Polling http://download01.phx2.fedoraproject.org/pub/fedora/linux/updates/testing/26/aarch64/repodata/r

[2017-03-10 09:54:58][    fedmsg   ERROR]f26-updates-testing Error fetching repomd.xml
Traceback (most recent call last):
  File "/usr/lib/python2.7/site-packages/bodhi/server/consumers/masher.py", line 772, in wait_for_sync
    masterrepomd = urllib2.urlopen(url)
  File "/usr/lib64/python2.7/urllib2.py", line 154, in urlopen
    return opener.open(url, data, timeout)
  File "/usr/lib64/python2.7/urllib2.py", line 435, in open
    response = meth(req, response)
  File "/usr/lib64/python2.7/urllib2.py", line 548, in http_response
    'http', request, response, code, msg, hdrs)
  File "/usr/lib64/python2.7/urllib2.py", line 473, in error
    return self._call_chain(*args)
  File "/usr/lib64/python2.7/urllib2.py", line 407, in _call_chain
    result = func(*args)
  File "/usr/lib64/python2.7/urllib2.py", line 556, in http_error_default
    raise HTTPError(req.get_full_url(), code, msg, hdrs, fp) 
HTTPError: HTTP Error 404: Not Found
```

The issue is that Bodhi is polling the wrong URL to find out when the ```repomd.xml``` file has been synchronized to the master mirror for ```aarch64```, so the URL listed in the above log is a 404. It should be looking here instead:

http://download01.phx2.fedoraproject.org/pub/fedora-secondary/updates/testing/26/aarch64/repodata/repomd.xml

Bodhi doesn't have any concept of primary and alternative architectures currently, but it's going to need to have some way to express which architectures are primary and alternative per release. Starting with Fedora 26, there are quite a few alternative architectures that are now handled by Bodhi, and ```i386``` has been switched from primary to alternative in this release as well.

We will need to add settings to production.ini for alternative arch mirror locations, and we will need to provide some way to express which architectures are primary vs. alternative.


Discussion:
The [following patch](http://pkgs.fedoraproject.org/cgit/rpms/bodhi.git/commit/?id=badf506c5efc381c2f47ddc1ddc9401da1649a7b) has been applied to production to workaround this issue:

```
From 72b062de6687502a9cd9498e70d2cd6c084527d0 Mon Sep 17 00:00:00 2001
From: Randy Barlow <randy@electronsweatshop.com>
Date: Fri, 10 Mar 2017 15:11:10 -0500
Subject: [PATCH] Hard code wait_for_sync() to look for x86_64.

---
 bodhi/server/consumers/masher.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/bodhi/server/consumers/masher.py b/bodhi/server/consumers/masher.py
index 38e1eec3..978963fb 100644
--- a/bodhi/server/consumers/masher.py
+++ b/bodhi/server/consumers/masher.py
@@ -750,7 +750,7 @@ class MasherThread(threading.Thread):
             force=True,
         )
         mash_path = os.path.join(self.path, self.id)
-        arch = os.listdir(mash_path)[0]
+        arch = 'x86_64'
 
         release = self.release.id_prefix.lower().replace('-', '_')
         request = self.request.value
-- 
2.12.0
```

Resolving patch:
diff --git a/bodhi/server/consumers/masher.py b/bodhi/server/consumers/masher.py
--- a/bodhi/server/consumers/masher.py
+++ b/bodhi/server/consumers/masher.py
@@ -752,24 +752,18 @@ def wait_for_sync(self):
         mash_path = os.path.join(self.path, self.id)
         arch = os.listdir(mash_path)[0]
 
-        release = self.release.id_prefix.lower().replace('-', '_')
-        request = self.request.value
-        key = '%s_%s_master_repomd' % (release, request)
-        master_repomd = config.get(key)
-        if not master_repomd:
-            raise ValueError("Could not find %s in the config file" % key)
-
         repomd = os.path.join(mash_path, arch, 'repodata', 'repomd.xml')
         if not os.path.exists(repomd):
             self.log.error('Cannot find local repomd: %s', repomd)
             return
 
+        master_repomd_url = self._get_master_repomd_url(arch)
+
         checksum = hashlib.sha1(file(repomd).read()).hexdigest()
         while True:
             try:
-                url = master_repomd % (self.release.version, arch)
-                self.log.info('Polling %s' % url)
-                masterrepomd = urllib2.urlopen(url)
+                self.log.info('Polling %s' % master_repomd_url)
+                masterrepomd = urllib2.urlopen(master_repomd_url)
             except (urllib2.URLError, urllib2.HTTPError):
                 self.log.exception('Error fetching repomd.xml')
                 time.sleep(200)
@@ -947,6 +941,38 @@ def compose_atomic_trees(self):
                              commitid=result.get('commitid')),
                     force=True)
 
+    def _get_master_repomd_url(self, arch):
+        """
+        Look up the correct *_master_repomd setting in the config and use it to form the URL that
+        wait_for_sync() will use to determine when the repository has been synchronized to the
+        master mirror.
+
+        Args:
+            arch (basestring): The architecture for which a URL needs to be formed.
+
+        Returns:
+            basestring: A URL on the master mirror where the repomd.xml file should be synchronized.
+        """
+        release = self.release.id_prefix.lower().replace('-', '_')
+        request = self.request.value
+
+        # If the release has primary_arches defined in the config, we need to consider whether to
+        # use the release's *alt_master_repomd setting.
+        primary_arches = config.get(
+            '{release}_{version}_primary_arches'.format(
+                release=release, version=self.release.version))
+        if primary_arches and arch not in primary_arches.split():
+            key = '%s_%s_alt_master_repomd'
+        else:
+            key = '%s_%s_master_repomd'
+        key = key % (release, request)
+
+        master_repomd = config.get(key)
+        if not master_repomd:
+            raise ValueError("Could not find %s in the config file" % key)
+
+        return master_repomd % (self.release.version, arch)
+
 
 class MashThread(threading.Thread):
 


Test patch:
diff --git a/bodhi/tests/server/consumers/test_masher.py b/bodhi/tests/server/consumers/test_masher.py
--- a/bodhi/tests/server/consumers/test_masher.py
+++ b/bodhi/tests/server/consumers/test_masher.py
@@ -11,6 +11,7 @@
 # You should have received a copy of the GNU General Public License
 # along with this program; if not, write to the Free Software
 # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
+from cStringIO import StringIO
 import datetime
 import json
 import os
@@ -18,6 +19,7 @@
 import tempfile
 import time
 import unittest
+import urllib2
 import urlparse
 
 from sqlalchemy import create_engine
@@ -1133,6 +1135,97 @@ def tearDown(self):
         buildsys.teardown_buildsystem()
 
 
+class TestMasherThread__get_master_repomd_url(MasherThreadBaseTestCase):
+    """This test class contains tests for the MasherThread._get_master_repomd_url() method."""
+    @mock.patch.dict(
+        'bodhi.server.consumers.masher.config',
+        {'fedora_17_primary_arches': 'armhfp x86_64',
+         'fedora_testing_master_repomd':
+            'http://example.com/pub/fedora/linux/updates/testing/%s/%s/repodata.repomd.xml',
+         'fedora_testing_alt_master_repomd':
+         'http://example.com/pub/fedora-secondary/updates/testing/%s/%s/repodata.repomd.xml'})
+    def test_alternative_arch(self):
+        """
+        Assert that the *_alt_master_repomd settings are used when the release does define primary
+        arches and the arch being looked up is not in the primary arch list.
+        """
+        release = self.db.query(Release).filter_by(name=u'F17').one()
+        t = MasherThread(release, u'testing', [u'bodhi-2.4.0-1.fc26'],
+                         'bowlofeggs', log, self.Session, self.tempdir)
+
+        url = t._get_master_repomd_url('aarch64')
+
+        self.assertEqual(
+            url,
+            'http://example.com/pub/fedora-secondary/updates/testing/17/aarch64/repodata.repomd.xml'
+        )
+
+    @mock.patch.dict(
+        'bodhi.server.consumers.masher.config',
+        {'fedora_17_primary_arches': 'armhfp x86_64',
+         'fedora_testing_master_repomd': None,
+         'fedora_testing_alt_master_repomd': None})
+    def test_master_repomd_undefined(self):
+        """
+        Assert that a ValueError is raised when the config is missing a master_repomd config for
+        the release.
+        """
+        release = self.db.query(Release).filter_by(name=u'F17').one()
+        t = MasherThread(release, u'testing', [u'bodhi-2.4.0-1.fc26'],
+                         'bowlofeggs', log, self.Session, self.tempdir)
+
+        with self.assertRaises(ValueError) as exc:
+            t._get_master_repomd_url('aarch64')
+
+        self.assertEqual(unicode(exc.exception),
+                         'Could not find fedora_testing_alt_master_repomd in the config file')
+
+    @mock.patch.dict(
+        'bodhi.server.consumers.masher.config',
+        {'fedora_17_primary_arches': 'armhfp x86_64',
+         'fedora_testing_master_repomd':
+            'http://example.com/pub/fedora/linux/updates/testing/%s/%s/repodata.repomd.xml',
+         'fedora_testing_alt_master_repomd':
+         'http://example.com/pub/fedora-secondary/updates/testing/%s/%s/repodata.repomd.xml'})
+    def test_primary_arch(self):
+        """
+        Assert that the *_master_repomd settings are used when the release does define primary
+        arches and the arch being looked up is primary.
+        """
+        release = self.db.query(Release).filter_by(name=u'F17').one()
+        t = MasherThread(release, u'testing', [u'bodhi-2.4.0-1.fc26'],
+                         'bowlofeggs', log, self.Session, self.tempdir)
+
+        url = t._get_master_repomd_url('x86_64')
+
+        self.assertEqual(
+            url,
+            'http://example.com/pub/fedora/linux/updates/testing/17/x86_64/repodata.repomd.xml'
+        )
+
+    @mock.patch.dict(
+        'bodhi.server.consumers.masher.config',
+        {'fedora_testing_master_repomd':
+            'http://example.com/pub/fedora/linux/updates/testing/%s/%s/repodata.repomd.xml',
+         'fedora_testing_alt_master_repomd':
+         'http://example.com/pub/fedora-secondary/updates/testing/%s/%s/repodata.repomd.xml'})
+    def test_primary_arches_undefined(self):
+        """
+        Assert that the *_master_repomd settings are used when the release does not have primary
+        arches defined in the config file.
+        """
+        release = self.db.query(Release).filter_by(name=u'F17').one()
+        t = MasherThread(release, u'testing', [u'bodhi-2.4.0-1.fc26'],
+                         'bowlofeggs', log, self.Session, self.tempdir)
+
+        url = t._get_master_repomd_url('aarch64')
+
+        self.assertEqual(
+            url,
+            'http://example.com/pub/fedora/linux/updates/testing/17/aarch64/repodata.repomd.xml'
+        )
+
+
 class TestMasherThread__perform_tag_actions(MasherThreadBaseTestCase):
     """This test class contains tests for the MasherThread._perform_tag_actions() method."""
     @mock.patch('bodhi.server.consumers.masher.buildsys.wait_for_tasks')
@@ -1231,3 +1324,220 @@ def test_comps_unsafe_http_url(self, mock_cmd):
         self.masher_thread.update_comps()
         self.assertEqual(0, mock_cmd.call_count)
         self.masher_thread.log.error.assert_called_once_with('comps_url must start with https://')
+
+
+class TestMasherThread_wait_for_sync(MasherThreadBaseTestCase):
+    """This test class contains tests for the MasherThread.wait_for_sync() method."""
+    @mock.patch.dict(
+        'bodhi.server.consumers.masher.config',
+        {'fedora_testing_master_repomd':
+            'http://example.com/pub/fedora/linux/updates/testing/%s/%s/repodata.repomd.xml'})
+    @mock.patch('bodhi.server.consumers.masher.notifications.publish')
+    @mock.patch('bodhi.server.consumers.masher.time.sleep',
+                mock.MagicMock(side_effect=Exception('This should not happen during this test.')))
+    @mock.patch('bodhi.server.consumers.masher.urllib2.urlopen',
+                return_value=StringIO('---\nyaml: rules'))
+    def test_checksum_match_immediately(self, urlopen, publish):
+        """
+        Assert correct operation when the repomd checksum matches immediately.
+        """
+        release = self.db.query(Release).filter_by(name=u'F17').one()
+        t = MasherThread(release, u'testing', [u'bodhi-2.4.0-1.fc26'],
+                         'bowlofeggs', log, self.Session, self.tempdir)
+        t.id = 'f26-updates-testing'
+        t.path = os.path.join(self.tempdir, t.id + '-' + time.strftime("%y%m%d.%H%M"))
+        for arch in ['aarch64', 'x86_64']:
+            repodata = os.path.join(t.path, t.id, arch, 'repodata')
+            os.makedirs(repodata)
+            with open(os.path.join(repodata, 'repomd.xml'), 'w') as repomd:
+                repomd.write('---\nyaml: rules')
+
+        t.wait_for_sync()
+
+        expected_calls = [
+            mock.call(topic='mashtask.sync.wait', msg={'repo': t.id, 'agent': 'bowlofeggs'},
+                      force=True),
+            mock.call(topic='mashtask.sync.done', msg={'repo': t.id, 'agent': 'bowlofeggs'},
+                      force=True)]
+        publish.assert_has_calls(expected_calls)
+        urlopen.assert_called_once_with(
+            'http://example.com/pub/fedora/linux/updates/testing/17/x86_64/repodata.repomd.xml')
+
+    @mock.patch.dict(
+        'bodhi.server.consumers.masher.config',
+        {'fedora_testing_master_repomd':
+            'http://example.com/pub/fedora/linux/updates/testing/%s/%s/repodata.repomd.xml'})
+    @mock.patch('bodhi.server.consumers.masher.notifications.publish')
+    @mock.patch('bodhi.server.consumers.masher.time.sleep')
+    @mock.patch(
+        'bodhi.server.consumers.masher.urllib2.urlopen',
+        side_effect=[StringIO('wrong'), StringIO('nope'), StringIO('---\nyaml: rules')])
+    def test_checksum_match_third_try(self, urlopen, sleep, publish):
+        """
+        Assert correct operation when the repomd checksum matches on the third try.
+        """
+        release = self.db.query(Release).filter_by(name=u'F17').one()
+        t = MasherThread(release, u'testing', [u'bodhi-2.4.0-1.fc26'],
+                         'bowlofeggs', log, self.Session, self.tempdir)
+        t.id = 'f26-updates-testing'
+        t.path = os.path.join(self.tempdir, t.id + '-' + time.strftime("%y%m%d.%H%M"))
+        for arch in ['aarch64', 'x86_64']:
+            repodata = os.path.join(t.path, t.id, arch, 'repodata')
+            os.makedirs(repodata)
+            with open(os.path.join(repodata, 'repomd.xml'), 'w') as repomd:
+                repomd.write('---\nyaml: rules')
+
+        t.wait_for_sync()
+
+        expected_calls = [
+            mock.call(topic='mashtask.sync.wait', msg={'repo': t.id, 'agent': 'bowlofeggs'},
+                      force=True),
+            mock.call(topic='mashtask.sync.done', msg={'repo': t.id, 'agent': 'bowlofeggs'},
+                      force=True)]
+        publish.assert_has_calls(expected_calls)
+        expected_calls = [
+            mock.call(
+                'http://example.com/pub/fedora/linux/updates/testing/17/x86_64/repodata.repomd.xml')
+            for i in range(3)]
+        urlopen.assert_has_calls(expected_calls)
+        sleep.assert_has_calls([mock.call(200), mock.call(200)])
+
+    @mock.patch.dict(
+        'bodhi.server.consumers.masher.config',
+        {'fedora_testing_master_repomd':
+            'http://example.com/pub/fedora/linux/updates/testing/%s/%s/repodata.repomd.xml'})
+    @mock.patch('bodhi.server.consumers.masher.notifications.publish')
+    @mock.patch('bodhi.server.consumers.masher.time.sleep')
+    @mock.patch(
+        'bodhi.server.consumers.masher.urllib2.urlopen',
+        side_effect=[urllib2.HTTPError('url', 404, 'Not found', {}, None),
+                     StringIO('---\nyaml: rules')])
+    def test_httperror(self, urlopen, sleep, publish):
+        """
+        Assert that an HTTPError is properly caught and logged, and that the algorithm continues.
+        """
+        release = self.db.query(Release).filter_by(name=u'F17').one()
+        t = MasherThread(release, u'testing', [u'bodhi-2.4.0-1.fc26'],
+                         'bowlofeggs', log, self.Session, self.tempdir)
+        t.id = 'f26-updates-testing'
+        t.log = mock.MagicMock()
+        t.path = os.path.join(self.tempdir, t.id + '-' + time.strftime("%y%m%d.%H%M"))
+        for arch in ['aarch64', 'x86_64']:
+            repodata = os.path.join(t.path, t.id, arch, 'repodata')
+            os.makedirs(repodata)
+            with open(os.path.join(repodata, 'repomd.xml'), 'w') as repomd:
+                repomd.write('---\nyaml: rules')
+
+        t.wait_for_sync()
+
+        expected_calls = [
+            mock.call(topic='mashtask.sync.wait', msg={'repo': t.id, 'agent': 'bowlofeggs'},
+                      force=True),
+            mock.call(topic='mashtask.sync.done', msg={'repo': t.id, 'agent': 'bowlofeggs'},
+                      force=True)]
+        publish.assert_has_calls(expected_calls)
+        expected_calls = [
+            mock.call(
+                'http://example.com/pub/fedora/linux/updates/testing/17/x86_64/repodata.repomd.xml')
+            for i in range(2)]
+        urlopen.assert_has_calls(expected_calls)
+        t.log.exception.assert_called_once_with('Error fetching repomd.xml')
+        sleep.assert_called_once_with(200)
+
+    @mock.patch.dict(
+        'bodhi.server.consumers.masher.config',
+        {'fedora_testing_master_repomd': None})
+    @mock.patch('bodhi.server.consumers.masher.notifications.publish')
+    @mock.patch('bodhi.server.consumers.masher.time.sleep',
+                mock.MagicMock(side_effect=Exception('This should not happen during this test.')))
+    @mock.patch('bodhi.server.consumers.masher.urllib2.urlopen',
+                mock.MagicMock(side_effect=Exception('urlopen should not be called')))
+    def test_missing_config_key(self, publish):
+        """
+        Assert that a ValueError is raised when the needed *_master_repomd config is missing.
+        """
+        release = self.db.query(Release).filter_by(name=u'F17').one()
+        t = MasherThread(release, u'testing', [u'bodhi-2.4.0-1.fc26'],
+                         'bowlofeggs', log, self.Session, self.tempdir)
+        t.id = 'f26-updates-testing'
+        t.path = os.path.join(self.tempdir, t.id + '-' + time.strftime("%y%m%d.%H%M"))
+        for arch in ['aarch64', 'x86_64']:
+            repodata = os.path.join(t.path, t.id, arch, 'repodata')
+            os.makedirs(repodata)
+            with open(os.path.join(repodata, 'repomd.xml'), 'w') as repomd:
+                repomd.write('---\nyaml: rules')
+
+        with self.assertRaises(ValueError) as exc:
+            t.wait_for_sync()
+
+        self.assertEqual(unicode(exc.exception),
+                         'Could not find fedora_testing_master_repomd in the config file')
+        publish.assert_called_once_with(topic='mashtask.sync.wait',
+                                        msg={'repo': t.id, 'agent': 'bowlofeggs'}, force=True)
+
+    @mock.patch('bodhi.server.consumers.masher.notifications.publish')
+    @mock.patch('bodhi.server.consumers.masher.time.sleep',
+                mock.MagicMock(side_effect=Exception('This should not happen during this test.')))
+    @mock.patch('bodhi.server.consumers.masher.urllib2.urlopen',
+                mock.MagicMock(side_effect=Exception('urlopen should not be called')))
+    def test_missing_repomd(self, publish):
+        """
+        Assert that an error is logged when the local repomd is missing.
+        """
+        release = self.db.query(Release).filter_by(name=u'F17').one()
+        t = MasherThread(release, u'testing', [u'bodhi-2.4.0-1.fc26'],
+                         'bowlofeggs', log, self.Session, self.tempdir)
+        t.id = 'f26-updates-testing'
+        t.log = mock.MagicMock()
+        t.path = os.path.join(self.tempdir, t.id + '-' + time.strftime("%y%m%d.%H%M"))
+        repodata = os.path.join(t.path, t.id, 'x86_64', 'repodata')
+        os.makedirs(repodata)
+
+        t.wait_for_sync()
+
+        publish.assert_called_once_with(topic='mashtask.sync.wait',
+                                        msg={'repo': t.id, 'agent': 'bowlofeggs'}, force=True)
+        t.log.error.assert_called_once_with(
+            'Cannot find local repomd: %s', os.path.join(repodata, 'repomd.xml'))
+
+    @mock.patch.dict(
+        'bodhi.server.consumers.masher.config',
+        {'fedora_testing_master_repomd':
+            'http://example.com/pub/fedora/linux/updates/testing/%s/%s/repodata.repomd.xml'})
+    @mock.patch('bodhi.server.consumers.masher.notifications.publish')
+    @mock.patch('bodhi.server.consumers.masher.time.sleep')
+    @mock.patch(
+        'bodhi.server.consumers.masher.urllib2.urlopen',
+        side_effect=[urllib2.URLError('it broke'),
+                     StringIO('---\nyaml: rules')])
+    def test_urlerror(self, urlopen, sleep, publish):
+        """
+        Assert that a URLError is properly caught and logged, and that the algorithm continues.
+        """
+        release = self.db.query(Release).filter_by(name=u'F17').one()
+        t = MasherThread(release, u'testing', [u'bodhi-2.4.0-1.fc26'],
+                         'bowlofeggs', log, self.Session, self.tempdir)
+        t.id = 'f26-updates-testing'
+        t.log = mock.MagicMock()
+        t.path = os.path.join(self.tempdir, t.id + '-' + time.strftime("%y%m%d.%H%M"))
+        for arch in ['aarch64', 'x86_64']:
+            repodata = os.path.join(t.path, t.id, arch, 'repodata')
+            os.makedirs(repodata)
+            with open(os.path.join(repodata, 'repomd.xml'), 'w') as repomd:
+                repomd.write('---\nyaml: rules')
+
+        t.wait_for_sync()
+
+        expected_calls = [
+            mock.call(topic='mashtask.sync.wait', msg={'repo': t.id, 'agent': 'bowlofeggs'},
+                      force=True),
+            mock.call(topic='mashtask.sync.done', msg={'repo': t.id, 'agent': 'bowlofeggs'},
+                      force=True)]
+        publish.assert_has_calls(expected_calls)
+        expected_calls = [
+            mock.call(
+                'http://example.com/pub/fedora/linux/updates/testing/17/x86_64/repodata.repomd.xml')
+            for i in range(2)]
+        urlopen.assert_has_calls(expected_calls)
+        t.log.exception.assert_called_once_with('Error fetching repomd.xml')
+        sleep.assert_called_once_with(200)