Repository: fedora-infra/bodhi

Issue:
Updates sometimes get stuck in pending state
From quick look, it seems that the composer does:
1. move from `f*-updates-candidate` to `f*-updates-testing`
2. do stuff
3. untag from `f*-updates-testing-pending`
4. mark update state as testing

If the composer hang on 2 the update remains stuck in pending as the builds are not tagged anymore in `f*-updates-candidate`. We should find a solution.


Discussion:

Resolving patch:
diff --git a/bodhi-server/bodhi/server/tasks/check_signed_builds.py b/bodhi-server/bodhi/server/tasks/check_signed_builds.py
--- a/bodhi-server/bodhi/server/tasks/check_signed_builds.py
+++ b/bodhi-server/bodhi/server/tasks/check_signed_builds.py
@@ -77,10 +77,16 @@ def main():
             pending_signing_tag = update.release.pending_signing_tag
             pending_testing_tag = update.release.pending_testing_tag
             for build in builds:
+                build_tags = [t['name'] for t in kc.listTags(build=build.nvr)]
                 if build.signed:
                     log.debug(f'{build.nvr} already marked as signed')
+                    if (update.release.testing_tag in build_tags
+                            and update.release.candidate_tag not in build_tags):
+                        # The update was probably ejected from a compose and is stuck
+                        log.debug(f'Resubmitting {update.alias} to testing')
+                        update.set_request(session, models.UpdateRequest.testing, 'bodhi')
+                        break
                     continue
-                build_tags = [t['name'] for t in kc.listTags(build=build.nvr)]
                 if pending_signing_tag not in build_tags and pending_testing_tag in build_tags:
                     # Our composer missed the message that the build got signed
                     log.debug(f'Changing signed status of {build.nvr}')


Test patch:
diff --git a/bodhi-server/tests/tasks/test_check_signed_builds.py b/bodhi-server/tests/tasks/test_check_signed_builds.py
--- a/bodhi-server/tests/tasks/test_check_signed_builds.py
+++ b/bodhi-server/tests/tasks/test_check_signed_builds.py
@@ -20,7 +20,7 @@
 """
 
 from datetime import datetime
-from unittest.mock import patch
+from unittest.mock import call, patch
 
 from bodhi.server import models
 from bodhi.server.tasks import check_signed_builds_task
@@ -104,6 +104,31 @@ def test_check_signed_builds_ignore_signed_builds(self, debug, buildsys):
         buildsys.get_session.assert_called_once()
         debug.assert_called_once_with('bodhi-2.0-1.fc17 already marked as signed')
 
+    @patch('bodhi.server.models.Update.set_request')
+    @patch('bodhi.server.tasks.check_signed_builds.buildsys')
+    @patch('bodhi.server.tasks.check_signed_builds.log.debug')
+    def test_check_signed_builds_stuck_Update_with_signed_build(self, debug, buildsys, request):
+        """
+        The update was probably ejected from a compose and is stuck.
+        """
+        update = models.Update.query.first()
+        assert update.builds[0].signed
+
+        self.db.commit()
+
+        listTags = [
+            {'arches': 'i386 x86_64 ppc ppc64', 'id': 10, 'locked': True,
+             'name': 'f17-updates-testing', 'perm': None, 'perm_id': None}, ]
+
+        buildsys.get_session.return_value.listTags.return_value = listTags
+        check_signed_builds_main()
+
+        buildsys.get_session.assert_called_once()
+        calls = [call('bodhi-2.0-1.fc17 already marked as signed'),
+                 call(f'Resubmitting {update.alias} to testing')]
+        debug.assert_has_calls(calls)
+        request.assert_called_once()
+
     @patch('bodhi.server.tasks.check_signed_builds.buildsys')
     @patch('bodhi.server.tasks.check_signed_builds.log.debug')
     def test_check_signed_builds_still_not_signed(self, debug, buildsys):