Repository: fedora-infra/bodhi

Issue:
Do not let update through when bodhi fails to talk to greenwave
Now that bodhi and greenwave have been together for a while and that we can see that appart from the unplanned outage, they are performing well. I believe it is time to stop letting bodhi push update through when it fails to talk to greenwave.

The current flow is as follow:
- bodhi queries greenwave
- if greenwave returns an error code or times-out
- bodhi sets the gating status to "failed"
- if there is a push, bodhi will take update that are in gating status "passed" and "failed", therefore letting through updates that should potentially be gated, but we do not know for sure because for some reason we failed to get a clear answer from greenwave.

There are two ways we can address this:
- Stop setting the gating status to "failed" when we fail to talk to greenwave (pros: less noise on the update, less notifications, less confusions to the users, cons: we don't know why the gating status has not been updated)
- Stop including updates with gating status "failed" in the push (pros/cons: the opposite as above)


Discussion:
In my opinion, if Bodhi fails to talk to Greenwave, the status should remain "waiting".

Also note that for updates in current Releases the status is set to "ignored" upon submission, then changes to "waiting" and then again to "ignored/passed/failed"... this needs to be fixed.

Resolving patch:
diff --git a/bodhi/server/models.py b/bodhi/server/models.py
--- a/bodhi/server/models.py
+++ b/bodhi/server/models.py
@@ -606,7 +606,6 @@ class TestGatingStatus(DeclEnum):
     running = 'running', 'Running'
     passed = 'passed', 'Passed'
     failed = 'failed', 'Failed'
-    greenwave_failed = 'greenwave_failed', 'Greenwave failed to respond'
 
 
 class UpdateType(DeclEnum):
@@ -2253,11 +2252,10 @@ def update_test_gating_status(self):
             self.test_gating_status = self._get_test_gating_status()
         except (requests.exceptions.Timeout, RuntimeError) as e:
             log.error(str(e))
-            # Greenwave frequently returns 500 response codes. When this happens, we do not want
-            # to block updates from proceeding, so we will consider this condition as having the
-            # policy satisfied. We will use the Exception as the summary so we can mark the status
-            # as ignored for the record.
-            self.test_gating_status = TestGatingStatus.greenwave_failed
+            # If we receive a 500 error code from Greenwave, we set the test_gating_status to
+            # waiting. The status will then be updated later by the greenwave fedora-messaging
+            # consumer.
+            self.test_gating_status = TestGatingStatus.waiting
 
     @classmethod
     def new(cls, request, data):
@@ -2555,11 +2553,10 @@ def test_gating_passed(self) -> bool:
 
         Returns:
             True if the Update's test_gating_status property is None,
-            greenwave_failed, ignored, or passed. Otherwise it returns False.
+            ignored, or passed. Otherwise it returns False.
         """
         if self.test_gating_status in (
-                None, TestGatingStatus.greenwave_failed, TestGatingStatus.ignored,
-                TestGatingStatus.passed):
+                None, TestGatingStatus.ignored, TestGatingStatus.passed):
             return True
         return False
 
@@ -3937,8 +3934,7 @@ def comment_on_test_gating_status_change(target, value, old, initiator):
         """
         Place comment on the update when ``test_gating_status`` changes.
 
-        Only notify the users by email if the new status is in ``failed`` or
-        ``greenwave_failed``.
+        Only notify the users by email if the new status is in ``failed``.
 
         Args:
             target (InstanceState): The state of the instance that has had a
@@ -3951,10 +3947,7 @@ def comment_on_test_gating_status_change(target, value, old, initiator):
         instance = target.object
 
         if value != old:
-            notify = value in [
-                TestGatingStatus.greenwave_failed,
-                TestGatingStatus.failed,
-            ]
+            notify = value == TestGatingStatus.failed
             instance.comment(
                 target.session,
                 f"This update's test gating status has been changed to '{value}'.",


Test patch:
diff --git a/bodhi/tests/server/test_models.py b/bodhi/tests/server/test_models.py
--- a/bodhi/tests/server/test_models.py
+++ b/bodhi/tests/server/test_models.py
@@ -2191,17 +2191,6 @@ def test_from_tag_update(self):
         assert not update.signed
 
 
-class TestUpdateTestGatingPassed(BasePyTestCase):
-    """Test the Update.test_gating_passed() method."""
-
-    def test_greenwave_failed(self):
-        """The greenwave_failed TestGatingStatus should count as passed."""
-        update = model.Update.query.first()
-        update.test_gating_status = TestGatingStatus.greenwave_failed
-
-        assert update.test_gating_passed
-
-
 class TestUpdateUpdateTestGatingStatus(BasePyTestCase):
     """Test the Update.update_test_gating_status() method."""
 
@@ -2219,7 +2208,7 @@ def test_500_response_from_greenwave(self, sleep, post, error):
 
         update.update_test_gating_status()
 
-        assert update.test_gating_status == model.TestGatingStatus.greenwave_failed
+        assert update.test_gating_status == model.TestGatingStatus.waiting
         assert sleep.mock_calls == [mock.call(1), mock.call(1), mock.call(1)]
         expected_post = mock.call(
             'https://greenwave-web-greenwave.app.os.fedoraproject.org/api/v1.0/decision',
@@ -2261,7 +2250,7 @@ def test_timeout_from_greenwave(self, sleep, post, error):
 
         update.update_test_gating_status()
 
-        assert update.test_gating_status == model.TestGatingStatus.greenwave_failed
+        assert update.test_gating_status == model.TestGatingStatus.waiting
         # The call_url() handler doesn't catch a Timeout so there are no sleeps/retries.
         assert sleep.mock_calls == []
         expected_post = mock.call(