Repository: fedora-infra/bodhi

Issue:
Add a ComposeState for waiting on the sync to mirrors
We should add a new ComposeState() to get set when ```_wait_for_sync()``` is called in the composer.


Discussion:

Resolving patch:
diff --git a/bodhi/server/consumers/masher.py b/bodhi/server/consumers/masher.py
--- a/bodhi/server/consumers/masher.py
+++ b/bodhi/server/consumers/masher.py
@@ -1308,6 +1308,7 @@ def _wait_for_sync(self):
             self.log.error('Cannot find local repomd: %s', repomd)
             return
 
+        self.save_state(ComposeState.syncing_repo)
         master_repomd_url = self._get_master_repomd_url(arch)
 
         with open(repomd) as repomdf:
diff --git a/bodhi/server/migrations/versions/68f9ccb1f388_add_a_syncing_repo_composestate_enum_.py b/bodhi/server/migrations/versions/68f9ccb1f388_add_a_syncing_repo_composestate_enum_.py
new file mode 100644
--- /dev/null
+++ b/bodhi/server/migrations/versions/68f9ccb1f388_add_a_syncing_repo_composestate_enum_.py
@@ -0,0 +1,54 @@
+# Copyright (c) 2018 Mattia Verga
+#
+# This file is part of Bodhi.
+#
+# This program is free software; you can redistribute it and/or
+# modify it under the terms of the GNU General Public License
+# as published by the Free Software Foundation; either version 2
+# of the License, or (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
+"""
+Add a syncing_repo ComposeState enum value.
+
+Revision ID: 68f9ccb1f388
+Revises: 385acbb51075
+Create Date: 2018-09-19 15:51:37.962897
+"""
+from alembic import op
+from sqlalchemy import exc
+
+
+# revision identifiers, used by Alembic.
+revision = '68f9ccb1f388'
+down_revision = '385acbb51075'
+
+
+def upgrade():
+    """Add a 'syncing_repo' content type enum value in ck_compose_state."""
+    op.execute('COMMIT')  # See https://bitbucket.org/zzzeek/alembic/issue/123
+    try:
+        # This will raise a ProgrammingError if the DB server doesn't use BDR.
+        op.execute('SHOW bdr.permit_ddl_locking')
+        # This server uses BDR, so let's ask for a DDL lock.
+        op.execute('SET LOCAL bdr.permit_ddl_locking = true')
+    except exc.ProgrammingError:
+        # This server doesn't use BDR, so no problem.
+        pass
+    op.execute("ALTER TYPE ck_compose_state ADD VALUE 'syncing_repo' AFTER 'punging'")
+
+
+def downgrade():
+    """
+    Alert user that we cannot downgrade this migration.
+
+    PostgreSQL does not allow enum values to be removed.
+    """
+    raise NotImplementedError("Downgrading this migration is not supported.")
diff --git a/bodhi/server/models.py b/bodhi/server/models.py
--- a/bodhi/server/models.py
+++ b/bodhi/server/models.py
@@ -747,6 +747,7 @@ class ComposeState(DeclEnum):
         initializing (EnumSymbol): The compose is initializing.
         updateinfo (EnumSymbol): The updateinfo.xml is being generated.
         punging (EnumSymbol): A Pungi soldier has been deployed to deal with the situation.
+        syncing_repo (EnumSymbol): The repo is being synced to the master mirror.
         notifying (EnumSymbol): Pungi has finished successfully, and we are now sending out various
             forms of notifications, such as e-mail, fedmsgs, and bugzilla.
         success (EnumSymbol): The Compose has completed successfully.
@@ -760,6 +761,7 @@ class ComposeState(DeclEnum):
     initializing = 'initializing', 'Initializing'
     updateinfo = 'updateinfo', 'Generating updateinfo.xml'
     punging = 'punging', 'Waiting for Pungi to finish'
+    syncing_repo = 'syncing_repo', 'Wait for the repo to hit the master mirror'
     notifying = 'notifying', 'Sending notifications'
     success = 'success', 'Success'
     failed = 'failed', 'Failed'


Test patch:
diff --git a/bodhi/tests/server/consumers/test_masher.py b/bodhi/tests/server/consumers/test_masher.py
--- a/bodhi/tests/server/consumers/test_masher.py
+++ b/bodhi/tests/server/consumers/test_masher.py
@@ -2887,12 +2887,13 @@ class TestPungiComposerThread__wait_for_sync(ComposerThreadBaseTestCase):
         'bodhi.server.consumers.masher.config',
         {'fedora_testing_master_repomd':
             'http://example.com/pub/fedora/linux/updates/testing/%s/%s/repodata.repomd.xml'})
+    @mock.patch('bodhi.server.consumers.masher.PungiComposerThread.save_state')
     @mock.patch('bodhi.server.consumers.masher.notifications.publish')
     @mock.patch('bodhi.server.consumers.masher.time.sleep',
                 mock.MagicMock(side_effect=Exception('This should not happen during this test.')))
     @mock.patch('bodhi.server.consumers.masher.urllib2.urlopen',
                 return_value=StringIO('---\nyaml: rules'))
-    def test_checksum_match_immediately(self, urlopen, publish):
+    def test_checksum_match_immediately(self, urlopen, publish, save):
         """
         Assert correct operation when the repomd checksum matches immediately.
         """
@@ -2925,17 +2926,19 @@ def test_checksum_match_immediately(self, urlopen, publish):
             mock.call('http://example.com/pub/fedora/linux/updates/testing/17/aarch64/'
                       'repodata.repomd.xml')]
         self.assertTrue(urlopen.mock_calls[0] in expected_calls)
+        save.assert_called_once_with(ComposeState.syncing_repo)
 
     @mock.patch.dict(
         'bodhi.server.consumers.masher.config',
         {'fedora_testing_master_repomd':
             'http://example.com/pub/fedora/linux/updates/testing/%s/%s/repodata.repomd.xml'})
+    @mock.patch('bodhi.server.consumers.masher.PungiComposerThread.save_state')
     @mock.patch('bodhi.server.consumers.masher.notifications.publish')
     @mock.patch('bodhi.server.consumers.masher.time.sleep',
                 mock.MagicMock(side_effect=Exception('This should not happen during this test.')))
     @mock.patch('bodhi.server.consumers.masher.urllib2.urlopen',
                 return_value=StringIO('---\nyaml: rules'))
-    def test_no_checkarch(self, urlopen, publish):
+    def test_no_checkarch(self, urlopen, publish, save):
         """
         Assert error when no checkarch is found.
         """
@@ -2955,17 +2958,19 @@ def test_no_checkarch(self, urlopen, publish):
             assert False, "Compose with just source passed"
         except Exception as ex:
             assert str(ex) == "Not found an arch to _wait_for_sync with"
+            save.assert_not_called()
 
     @mock.patch.dict(
         'bodhi.server.consumers.masher.config',
         {'fedora_testing_master_repomd':
             'http://example.com/pub/fedora/linux/updates/testing/%s/%s/repodata.repomd.xml'})
+    @mock.patch('bodhi.server.consumers.masher.PungiComposerThread.save_state')
     @mock.patch('bodhi.server.consumers.masher.notifications.publish')
     @mock.patch('bodhi.server.consumers.masher.time.sleep')
     @mock.patch(
         'bodhi.server.consumers.masher.urllib2.urlopen',
         side_effect=[StringIO('wrong'), StringIO('nope'), StringIO('---\nyaml: rules')])
-    def test_checksum_match_third_try(self, urlopen, sleep, publish):
+    def test_checksum_match_third_try(self, urlopen, sleep, publish, save):
         """
         Assert correct operation when the repomd checksum matches on the third try.
         """
@@ -2998,18 +3003,20 @@ def test_checksum_match_third_try(self, urlopen, sleep, publish):
             for i in range(3)]
         urlopen.assert_has_calls(expected_calls)
         sleep.assert_has_calls([mock.call(200), mock.call(200)])
+        save.assert_called_with(ComposeState.syncing_repo)
 
     @mock.patch.dict(
         'bodhi.server.consumers.masher.config',
         {'fedora_testing_master_repomd':
             'http://example.com/pub/fedora/linux/updates/testing/%s/%s/repodata.repomd.xml'})
+    @mock.patch('bodhi.server.consumers.masher.PungiComposerThread.save_state')
     @mock.patch('bodhi.server.consumers.masher.notifications.publish')
     @mock.patch('bodhi.server.consumers.masher.time.sleep')
     @mock.patch(
         'bodhi.server.consumers.masher.urllib2.urlopen',
         side_effect=[HTTPError('url', 404, 'Not found', {}, None),
                      StringIO('---\nyaml: rules')])
-    def test_httperror(self, urlopen, sleep, publish):
+    def test_httperror(self, urlopen, sleep, publish, save):
         """
         Assert that an HTTPError is properly caught and logged, and that the algorithm continues.
         """
@@ -3044,16 +3051,18 @@ def test_httperror(self, urlopen, sleep, publish):
         urlopen.assert_has_calls(expected_calls)
         t.log.exception.assert_called_once_with('Error fetching repomd.xml')
         sleep.assert_called_once_with(200)
+        save.assert_called_once_with(ComposeState.syncing_repo)
 
     @mock.patch.dict(
         'bodhi.server.consumers.masher.config',
         {'fedora_testing_master_repomd': None})
+    @mock.patch('bodhi.server.consumers.masher.PungiComposerThread.save_state')
     @mock.patch('bodhi.server.consumers.masher.notifications.publish')
     @mock.patch('bodhi.server.consumers.masher.time.sleep',
                 mock.MagicMock(side_effect=Exception('This should not happen during this test.')))
     @mock.patch('bodhi.server.consumers.masher.urllib2.urlopen',
                 mock.MagicMock(side_effect=Exception('urlopen should not be called')))
-    def test_missing_config_key(self, publish):
+    def test_missing_config_key(self, publish, save):
         """
         Assert that a ValueError is raised when the needed *_master_repomd config is missing.
         """
@@ -3076,13 +3085,15 @@ def test_missing_config_key(self, publish):
                          'fedora_testing_master_repomd in the config file')
         publish.assert_called_once_with(topic='mashtask.sync.wait',
                                         msg={'repo': t.id, 'agent': 'bowlofeggs'}, force=True)
+        save.assert_called_once_with(ComposeState.syncing_repo)
 
+    @mock.patch('bodhi.server.consumers.masher.PungiComposerThread.save_state')
     @mock.patch('bodhi.server.consumers.masher.notifications.publish')
     @mock.patch('bodhi.server.consumers.masher.time.sleep',
                 mock.MagicMock(side_effect=Exception('This should not happen during this test.')))
     @mock.patch('bodhi.server.consumers.masher.urllib2.urlopen',
                 mock.MagicMock(side_effect=Exception('urlopen should not be called')))
-    def test_missing_repomd(self, publish):
+    def test_missing_repomd(self, publish, save):
         """
         Assert that an error is logged when the local repomd is missing.
         """
@@ -3101,18 +3112,20 @@ def test_missing_repomd(self, publish):
                                         msg={'repo': t.id, 'agent': 'bowlofeggs'}, force=True)
         t.log.error.assert_called_once_with(
             'Cannot find local repomd: %s', os.path.join(repodata, 'repomd.xml'))
+        save.assert_not_called()
 
     @mock.patch.dict(
         'bodhi.server.consumers.masher.config',
         {'fedora_testing_master_repomd':
             'http://example.com/pub/fedora/linux/updates/testing/%s/%s/repodata.repomd.xml'})
+    @mock.patch('bodhi.server.consumers.masher.PungiComposerThread.save_state')
     @mock.patch('bodhi.server.consumers.masher.notifications.publish')
     @mock.patch('bodhi.server.consumers.masher.time.sleep')
     @mock.patch(
         'bodhi.server.consumers.masher.urllib2.urlopen',
         side_effect=[URLError('it broke'),
                      StringIO('---\nyaml: rules')])
-    def test_urlerror(self, urlopen, sleep, publish):
+    def test_urlerror(self, urlopen, sleep, publish, save):
         """
         Assert that a URLError is properly caught and logged, and that the algorithm continues.
         """
@@ -3147,6 +3160,7 @@ def test_urlerror(self, urlopen, sleep, publish):
         urlopen.assert_has_calls(expected_calls)
         t.log.exception.assert_called_once_with('Error fetching repomd.xml')
         sleep.assert_called_once_with(200)
+        save.assert_called_once_with(ComposeState.syncing_repo)
 
 
 class TestComposerThread__mark_status_changes(ComposerThreadBaseTestCase):