Repository: fedora-infra/bodhi

Issue:
Bodhi should only block updates when Greenwave returns a NOGO response
Currently Bodhi will only allow updates to proceed to repositories if Greenwave gives back a GO response. This means that updates will be blocked from proceeding if Greenwave raises an Exception, which has been [happening often recently](https://pagure.io/greenwave/issue/388). Due to the frequent nature of these errors, it is most sensible to adjust Bodhi such that it will only block an update if Greenwave gives back a positive NOGO answer, and that any other answer (GO, error, etc.) means that the update can proceed.


Discussion:

Resolving patch:
diff --git a/bodhi/server/migrations/versions/8c4d6aad9b78_add_the_greenwave_failed_enum_to_.py b/bodhi/server/migrations/versions/8c4d6aad9b78_add_the_greenwave_failed_enum_to_.py
new file mode 100644
--- /dev/null
+++ b/bodhi/server/migrations/versions/8c4d6aad9b78_add_the_greenwave_failed_enum_to_.py
@@ -0,0 +1,69 @@
+# Copyright (c) 2019 Red Hat, Inc.
+#
+# This file is part of Bodhi.
+#
+# This program is free software; you can redistribute it and/or
+# modify it under the terms of the GNU General Public License
+# as published by the Free Software Foundation; either version 2
+# of the License, or (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
+"""
+Add the greenwave_failed enum to TestGatingStatus.
+
+Revision ID: 8c4d6aad9b78
+Revises: aae0d29d49b7
+Create Date: 2019-02-26 22:38:15.420477
+"""
+from alembic import op
+from sqlalchemy import exc
+
+
+# revision identifiers, used by Alembic.
+revision = '8c4d6aad9b78'
+down_revision = 'aae0d29d49b7'
+
+
+def upgrade():
+    """Add the greenwave_failed enum to ck_test_gating_status."""
+    op.execute('COMMIT')  # See https://bitbucket.org/zzzeek/alembic/issue/123
+    try:
+        # This will raise a ProgrammingError if the DB server doesn't use BDR.
+        op.execute('SHOW bdr.permit_ddl_locking')
+        # This server uses BDR, so let's ask for a DDL lock.
+        op.execute('SET LOCAL bdr.permit_ddl_locking = true')
+    except exc.ProgrammingError:
+        # This server doesn't use BDR, so no problem.
+        pass
+    op.execute("ALTER TYPE ck_test_gating_status ADD VALUE 'greenwave_failed' AFTER 'failed'")
+
+
+def downgrade():
+    """Remove the greenwave_failed enum from ck_test_gating_status."""
+    op.execute('COMMIT')  # See https://bitbucket.org/zzzeek/alembic/issue/123
+    try:
+        # This will raise a ProgrammingError if the DB server doesn't use BDR.
+        op.execute('SHOW bdr.permit_ddl_locking')
+        # This server uses BDR, so let's ask for a DDL lock.
+        op.execute('SET LOCAL bdr.permit_ddl_locking = true')
+    except exc.ProgrammingError:
+        # This server doesn't use BDR, so no problem.
+        pass
+    op.execute(
+        ("UPDATE updates SET test_gating_status = 'failed' "
+         "WHERE test_gating_status = 'greenwave_failed'"))
+    op.execute("ALTER TYPE ck_test_gating_status RENAME TO ck_test_gating_status_old")
+    op.execute(
+        "CREATE TYPE ck_test_gating_status AS ENUM('ignored', 'queued', "
+        "'running', 'passed', 'failed', 'waiting')")
+    op.execute(
+        "ALTER TABLE updates ALTER COLUMN test_gating_status TYPE ck_test_gating_status "
+        "USING test_gating_status::text::ck_test_gating_status")
+    op.execute("DROP TYPE ck_test_gating_status_old")
diff --git a/bodhi/server/models.py b/bodhi/server/models.py
--- a/bodhi/server/models.py
+++ b/bodhi/server/models.py
@@ -38,6 +38,7 @@
 from sqlalchemy.orm.properties import RelationshipProperty
 from sqlalchemy.sql import text
 from sqlalchemy.types import SchemaType, TypeDecorator, Enum
+import requests.exceptions
 
 from bodhi.server import bugs, buildsys, log, mail, notifications, Session, util
 from bodhi.server.config import config
@@ -608,6 +609,7 @@ class TestGatingStatus(DeclEnum):
     running = 'running', 'Running'
     passed = 'passed', 'Passed'
     failed = 'failed', 'Failed'
+    greenwave_failed = 'greenwave_failed', 'Greenwave failed to respond'
 
 
 class UpdateType(DeclEnum):
@@ -1924,11 +1926,23 @@ def get_test_gating_info(self):
 
     def update_test_gating_status(self):
         """Query Greenwave about this update and set the test_gating_status as appropriate."""
-        decision = self.get_test_gating_info()
+        try:
+            decision = self.get_test_gating_info()
+            decision['greenwave_success'] = True
+        except (requests.exceptions.Timeout, RuntimeError) as e:
+            log.error(str(e))
+            # Greenwave frequently returns 500 response codes. When this happens, we do not want
+            # to block updates from proceeding, so we will consider this condition as having the
+            # policy satisfied. We will use the Exception as the summary so we can mark the status
+            # as ignored for the record.
+            decision = {'policies_satisfied': True, 'summary': str(e), 'greenwave_success': False}
         if decision['policies_satisfied']:
-            # If an unrestricted policy is applied and no tests are required
-            # on this update, let's set the test gating as ignored in Bodhi.
-            if decision['summary'] == 'no tests are required':
+            if not decision['greenwave_success']:
+                # Greenwave failed to respond, so let's mark this as ignored.
+                self.test_gating_status = TestGatingStatus.greenwave_failed
+            elif decision['summary'] == 'no tests are required':
+                # If an unrestricted policy is applied and no tests are required
+                # on this update, let's set the test gating as ignored in Bodhi.
                 self.test_gating_status = TestGatingStatus.ignored
             else:
                 self.test_gating_status = TestGatingStatus.passed
@@ -2173,16 +2187,17 @@ def content_type(self):
             return self.builds[0].type
 
     @property
-    def test_gating_passed(self):
+    def test_gating_passed(self) -> bool:
         """
         Returns a boolean representing if this update has passed the test gating.
 
         Returns:
-            bool: Returns True if the Update's test_gating_status property is None, ignored,
-                or passed. Otherwise it returns False.
+            True if the Update's test_gating_status property is None,
+            greenwave_failed, ignored, or passed. Otherwise it returns False.
         """
         if self.test_gating_status in (
-                None, TestGatingStatus.ignored, TestGatingStatus.passed):
+                None, TestGatingStatus.greenwave_failed, TestGatingStatus.ignored,
+                TestGatingStatus.passed):
             return True
         return False
 


Test patch:
diff --git a/bodhi/tests/server/test_models.py b/bodhi/tests/server/test_models.py
--- a/bodhi/tests/server/test_models.py
+++ b/bodhi/tests/server/test_models.py
@@ -28,6 +28,7 @@
 from pyramid.testing import DummyRequest
 from sqlalchemy.exc import IntegrityError
 import cornice
+import requests.exceptions
 
 from bodhi.server import models as model, buildsys, mail, util, Session
 from bodhi.server.config import config
@@ -1673,6 +1674,112 @@ def test_release_without_pending_signing_tag(self):
         self.assertTrue(update.signed)
 
 
+class TestUpdateTestGatingPassed(BaseTestCase):
+    """Test the Update.test_gating_passed() method."""
+
+    def test_greenwave_failed(self):
+        """The greenwave_failed TestGatingStatus should count as passed."""
+        update = model.Update.query.first()
+        update.test_gating_status = TestGatingStatus.greenwave_failed
+
+        self.assertTrue(update.test_gating_passed)
+
+
+class TestUpdateUpdateTestGatingStatus(BaseTestCase):
+    """Test the Update.update_test_gating_status() method."""
+
+    @mock.patch('bodhi.server.models.log.error')
+    @mock.patch('bodhi.server.util.http_session.post')
+    @mock.patch('bodhi.server.util.time.sleep')
+    def test_500_response_from_greenwave(self, sleep, post, error):
+        """A 500 response from Greenwave should result in marking the test results as ignored."""
+        post.return_value = mock.MagicMock()
+        post.return_value.status_code = 500
+        update = model.Update.query.first()
+        # Let's set this to anything other than ignored, so we can assert that
+        # update_test_gating_status() toggles it back.
+        update.test_gating_status = model.TestGatingStatus.passed
+
+        update.update_test_gating_status()
+
+        self.assertEqual(update.test_gating_status, model.TestGatingStatus.greenwave_failed)
+        self.assertEqual(
+            update.greenwave_summary_string,
+            ('Bodhi failed to send POST request to Greenwave at the following URL '
+             '"https://greenwave-web-greenwave.app.os.fedoraproject.org/api/v1.0/decision". The '
+             'status code was "500".'))
+        self.assertEqual(sleep.mock_calls, [mock.call(1), mock.call(1), mock.call(1)])
+        expected_post = mock.call(
+            'https://greenwave-web-greenwave.app.os.fedoraproject.org/api/v1.0/decision',
+            data={"product_version": "fedora-17", "decision_context": "bodhi_update_push_testing",
+                  "subject": [{"item": f"{update.builds[0].nvr}", "type": "koji_build"},
+                              {"original_spec_nvr": f"{update.builds[0].nvr}"},
+                              {"item": f"{update.alias}", "type": "bodhi_update"}],
+                  "verbose": True},
+            headers={'Content-Type': 'application/json'}, timeout=60)
+        self.assertEqual(post.call_count, 4)
+        for i in range(4):
+            # Make sure the positional arguments are correct.
+            self.assertEqual(post.mock_calls[i][1], expected_post[1])
+            self.assertEqual(post.mock_calls[i][2].keys(), expected_post[2].keys())
+            # The request has serialized our data as JSON. We should probably not just serialize our
+            # expected JSON, because we don't have a guarantee that it will serialize to the same
+            # string. So instead, let's deserialize the JSON that the mock captured and compare it
+            # to our dictionary above.
+            self.assertEqual(json.loads(post.mock_calls[i][2]['data']), expected_post[2]['data'])
+            # Make sure the other stuff is all the same
+            for key in expected_post[2].keys():
+                if key != 'data':
+                    self.assertEqual(post.mock_calls[i][2][key], expected_post[2][key])
+        self.assertEqual(
+            error.mock_calls,
+            [mock.call((
+                'Bodhi failed to send POST request to Greenwave at the following URL '
+                '"https://greenwave-web-greenwave.app.os.fedoraproject.org/api/v1.0/decision". The '
+                'status code was "500".')) for i in range(2)])
+
+    @mock.patch('bodhi.server.models.log.error')
+    @mock.patch('bodhi.server.util.http_session.post')
+    @mock.patch('bodhi.server.util.time.sleep')
+    def test_timeout_from_greenwave(self, sleep, post, error):
+        """Similar to the 500 test above, a timeout should also result in marking tests ignored."""
+        post.side_effect = requests.exceptions.ConnectTimeout('The connection timed out.')
+        update = model.Update.query.first()
+        # Let's set this to anything other than ignored, so we can assert that
+        # update_test_gating_status() toggles it back.
+        update.test_gating_status = model.TestGatingStatus.passed
+
+        update.update_test_gating_status()
+
+        self.assertEqual(update.test_gating_status, model.TestGatingStatus.greenwave_failed)
+        self.assertEqual(update.greenwave_summary_string, 'The connection timed out.')
+        # The call_url() handler doesn't catch a Timeout so there are no sleeps/retries.
+        self.assertEqual(sleep.mock_calls, [])
+        expected_post = mock.call(
+            'https://greenwave-web-greenwave.app.os.fedoraproject.org/api/v1.0/decision',
+            data={"product_version": "fedora-17", "decision_context": "bodhi_update_push_testing",
+                  "subject": [{"item": f"{update.builds[0].nvr}", "type": "koji_build"},
+                              {"original_spec_nvr": f"{update.builds[0].nvr}"},
+                              {"item": f"{update.alias}", "type": "bodhi_update"}],
+                  "verbose": True},
+            headers={'Content-Type': 'application/json'}, timeout=60)
+        self.assertEqual(post.call_count, 1)
+        # Make sure the positional arguments are correct.
+        self.assertEqual(post.mock_calls[0][1], expected_post[1])
+        self.assertEqual(post.mock_calls[0][2].keys(), expected_post[2].keys())
+        # The request has serialized our data as JSON. We should probably not just serialize our
+        # expected JSON, because we don't have a guarantee that it will serialize to the same
+        # string. So instead, let's deserialize the JSON that the mock captured and compare it
+        # to our dictionary above.
+        self.assertEqual(json.loads(post.mock_calls[0][2]['data']), expected_post[2]['data'])
+        # Make sure the other stuff is all the same
+        for key in expected_post[2].keys():
+            if key != 'data':
+                self.assertEqual(post.mock_calls[0][2][key], expected_post[2][key])
+        self.assertEqual(
+            error.mock_calls, [mock.call('The connection timed out.')])
+
+
 class TestUpdateValidateBuilds(BaseTestCase):
     """Tests for the :class:`Update` validator for builds."""