Repository: fedora-infra/bodhi

Issue:
Bodhi does not gate updates based on Greenwave when updates have non-truthy requirements
[```Update.check_requirements()```](https://github.com/fedora-infra/bodhi/blob/3.6.1/bodhi/server/models.py#L3165-L3248) returns True if ```Update.requirements``` is non-truthy. In Fedora's production Bodhi, updates do not have requirements by default, which means that the [greenwave decision check](https://github.com/fedora-infra/bodhi/blob/3.6.1/bodhi/server/models.py#L3243-L3244) is never reached unless developers type something into the update's requirements field. Since most developers do not do this, Fedora's Bodhi instance is effectively not gating any updates based on test results.


Discussion:

Resolving patch:
diff --git a/bodhi/server/models.py b/bodhi/server/models.py
--- a/bodhi/server/models.py
+++ b/bodhi/server/models.py
@@ -3199,6 +3199,9 @@ def check_requirements(self, session, settings):
             tuple: A tuple containing (result, reason) where result is a bool
                 and reason is a str.
         """
+        if config.get('test_gating.required') and not self.test_gating_passed:
+            return (False, "Required tests did not pass on this update")
+
         requirements = tokenize(self.requirements or '')
         requirements = list(requirements)
 
@@ -3266,9 +3269,6 @@ def check_requirements(self, session, settings):
                     return False, "Required task %s returned %s" % (
                         latest['testcase']['name'], latest['outcome'])
 
-        if config.get('test_gating.required') and not self.test_gating_passed:
-            return (False, "Required tests did not pass on this update")
-
         # TODO - check require_bugs and require_testcases also?
 
         return True, "All checks pass."


Test patch:
diff --git a/bodhi/tests/server/consumers/test_masher.py b/bodhi/tests/server/consumers/test_masher.py
--- a/bodhi/tests/server/consumers/test_masher.py
+++ b/bodhi/tests/server/consumers/test_masher.py
@@ -1377,6 +1377,92 @@ def test_failed_gating(self, publish, *args):
              'send_testing_digest': True,
              'status_comments': True})
 
+    @mock.patch.dict(config, {'test_gating.required': True})
+    @mock.patch('bodhi.server.consumers.masher.PungiComposerThread._sanity_check_repo')
+    @mock.patch('bodhi.server.consumers.masher.PungiComposerThread._stage_repo')
+    @mock.patch('bodhi.server.consumers.masher.PungiComposerThread._wait_for_repo_signature')
+    @mock.patch('bodhi.server.consumers.masher.PungiComposerThread._generate_updateinfo')
+    @mock.patch('bodhi.server.consumers.masher.PungiComposerThread._wait_for_sync')
+    @mock.patch('bodhi.server.consumers.masher.time.sleep', return_value=None)
+    @mock.patch('bodhi.server.notifications.publish')
+    def test_test_gating_status_failed(self, publish, *args):
+        """If the update's test_gating_status is failed it should be ejected."""
+        self.expected_sems = 1
+        # Set the request to stable right out the gate so we can test gating
+        self.set_stable_request(u'bodhi-2.0-1.fc17')
+        u = Update.query.filter_by(title='bodhi-2.0-1.fc17').one()
+        u.test_gating_status = TestGatingStatus.failed
+        u.requirements = ''
+        msg = self._make_msg()
+        t = RPMComposerThread(
+            self.semmock, msg['body']['msg']['composes'][0], 'ralph', log, self.db_factory,
+            self.tempdir)
+
+        with self.db_factory() as session:
+            with mock.patch('bodhi.server.consumers.masher.subprocess.Popen') as Popen:
+                release = session.query(Release).filter_by(name=u'F17').one()
+                Popen.side_effect = self._generate_fake_pungi(t, 'stable_tag', release)
+                t.run()
+
+        # Also, ensure we reported success
+        publish.assert_called_with(topic="mashtask.complete",
+                                   force=True,
+                                   msg=dict(success=True,
+                                            ctype='rpm',
+                                            repo='f17-updates',
+                                            agent='ralph'))
+        publish.assert_any_call(topic='update.eject', msg=mock.ANY, force=True)
+        u = Update.query.filter_by(title='bodhi-2.0-1.fc17').one()
+        self.assertEqual(
+            u.comments[-1].text,
+            ("bodhi-2.0-1.fc17 ejected from the push because 'Required tests did not pass on this "
+             "update'"))
+        # The request got sent back to None since it was ejected.
+        self.assertEqual(u.request, None)
+        self.assertEqual(u.status, UpdateStatus.pending)
+
+    @mock.patch.dict(config, {'test_gating.required': True})
+    @mock.patch('bodhi.server.consumers.masher.PungiComposerThread._sanity_check_repo')
+    @mock.patch('bodhi.server.consumers.masher.PungiComposerThread._stage_repo')
+    @mock.patch('bodhi.server.consumers.masher.PungiComposerThread._wait_for_repo_signature')
+    @mock.patch('bodhi.server.consumers.masher.PungiComposerThread._generate_updateinfo')
+    @mock.patch('bodhi.server.consumers.masher.PungiComposerThread._wait_for_sync')
+    @mock.patch('bodhi.server.consumers.masher.time.sleep', return_value=None)
+    @mock.patch('bodhi.server.notifications.publish')
+    def test_test_gating_status_passed(self, publish, *args):
+        """If the update's test_gating_status is passed it should not be ejected."""
+        self.expected_sems = 1
+        # Set the request to stable right out the gate so we can test gating
+        self.set_stable_request(u'bodhi-2.0-1.fc17')
+        u = Update.query.filter_by(title='bodhi-2.0-1.fc17').one()
+        u.test_gating_status = TestGatingStatus.passed
+        u.requirements = ''
+        msg = self._make_msg()
+        t = RPMComposerThread(
+            self.semmock, msg['body']['msg']['composes'][0], 'ralph', log, self.db_factory,
+            self.tempdir)
+
+        with self.db_factory() as session:
+            with mock.patch('bodhi.server.consumers.masher.subprocess.Popen') as Popen:
+                release = session.query(Release).filter_by(name=u'F17').one()
+                Popen.side_effect = self._generate_fake_pungi(t, 'stable_tag', release)
+                t.run()
+
+        # Also, ensure we reported success
+        publish.assert_called_with(topic="mashtask.complete",
+                                   force=True,
+                                   msg=dict(success=True,
+                                            ctype='rpm',
+                                            repo='f17-updates',
+                                            agent='ralph'))
+        topics = {c[2]['topic'] for c in publish.mock_calls}
+        self.assertNotIn('update.eject', topics)
+        u = Update.query.filter_by(title='bodhi-2.0-1.fc17').one()
+        self.assertEqual(u.comments[-1].text, 'This update has been pushed to stable.')
+        # The update should be stable.
+        self.assertEqual(u.request, None)
+        self.assertEqual(u.status, UpdateStatus.stable)
+
     @mock.patch(**mock_absent_taskotron_results)
     @mock.patch('bodhi.server.consumers.masher.PungiComposerThread._sanity_check_repo')
     @mock.patch('bodhi.server.consumers.masher.PungiComposerThread._stage_repo')
diff --git a/bodhi/tests/server/services/test_updates.py b/bodhi/tests/server/services/test_updates.py
--- a/bodhi/tests/server/services/test_updates.py
+++ b/bodhi/tests/server/services/test_updates.py
@@ -698,6 +698,43 @@ def test_set_request_archived_release(self, *args):
         self.assertEquals(res.json_body[u'errors'][0][u'description'],
                           "Can't change request for an archived release")
 
+    @mock.patch.dict(config, {'test_gating.required': True})
+    def test_test_gating_status_failed(self):
+        """If the update's test_gating_status is failed, a user should not be able to push."""
+        nvr = u'bodhi-2.0-1.fc17'
+        up = self.db.query(Update).filter_by(title=nvr).one()
+        up.locked = False
+        up.requirements = ''
+        up.test_gating_status = TestGatingStatus.failed
+        up.date_testing = datetime.utcnow() - timedelta(days=8)
+        up.request = None
+        post_data = dict(update=nvr, request='stable', csrf_token=self.get_csrf_token())
+
+        res = self.app.post_json('/updates/%s/request' % str(nvr), post_data, status=400)
+
+        up = self.db.query(Update).filter_by(title=nvr).one()
+        self.assertEqual(up.request, None)
+        self.assertEquals(res.json_body['status'], 'error')
+        self.assertEquals(res.json_body[u'errors'][0][u'description'],
+                          "Requirement not met Required tests did not pass on this update")
+
+    @mock.patch.dict(config, {'test_gating.required': True})
+    def test_test_gating_status_passed(self):
+        """If the update's test_gating_status is passed, a user should be able to push."""
+        nvr = u'bodhi-2.0-1.fc17'
+        up = self.db.query(Update).filter_by(title=nvr).one()
+        up.locked = False
+        up.requirements = ''
+        up.test_gating_status = TestGatingStatus.passed
+        up.date_testing = datetime.utcnow() - timedelta(days=8)
+        post_data = dict(update=nvr, request='stable', csrf_token=self.get_csrf_token())
+
+        res = self.app.post_json('/updates/%s/request' % str(nvr), post_data, status=200)
+
+        up = self.db.query(Update).filter_by(title=nvr).one()
+        self.assertEqual(up.request, UpdateRequest.stable)
+        self.assertEqual(res.json['update']['request'], 'stable')
+
     @unittest.skipIf(six.PY3, 'Not working with Python 3 yet')
     @mock.patch(**mock_valid_requirements)
     @mock.patch('bodhi.server.services.updates.Update.set_request',
@@ -4296,6 +4333,58 @@ def test_edit_button_not_present_when_stable(self, publish, *args):
         self.assertNotIn('Push to Stable', resp)
         self.assertNotIn('Edit', resp)
 
+    @mock.patch.dict('bodhi.server.models.config', {'test_gating.required': True})
+    @mock.patch('bodhi.server.notifications.publish')
+    def test_push_to_batched_button_not_present_when_test_gating_status_failed(self, publish):
+        """The push to batched button should not appear if the test_gating_status is failed."""
+        nvr = u'bodhi-2.0.0-2.fc17'
+        args = self.get_update(nvr)
+        args['requirements'] = ''
+        resp = self.app.post_json('/updates/', args, headers={'Accept': 'application/json'})
+        update = Update.get(nvr)
+        update.status = UpdateStatus.testing
+        update.request = None
+        update.pushed = True
+        update.autokarma = False
+        update.stable_karma = 1
+        update.test_gating_status = TestGatingStatus.failed
+        update.comment(self.db, 'works', 1, 'bowlofeggs')
+        self.db.commit()
+        self.app_settings['test_gating.required'] = True
+        app = TestApp(main({}, testing=u'guest', **self.app_settings))
+
+        resp = app.get('/updates/%s' % nvr, headers={'Accept': 'text/html'})
+
+        self.assertNotIn('Push to Batched', resp)
+        self.assertNotIn('Push to Stable', resp)
+        self.assertIn('Edit', resp)
+
+    @mock.patch.dict('bodhi.server.models.config', {'test_gating.required': True})
+    @mock.patch('bodhi.server.notifications.publish')
+    def test_push_to_batched_button_present_when_test_gating_status_passed(self, publish):
+        """The push to batched button should appear if the test_gating_status is passed."""
+        nvr = u'bodhi-2.0.0-2.fc17'
+        args = self.get_update(nvr)
+        args['requirements'] = ''
+        resp = self.app.post_json('/updates/', args, headers={'Accept': 'application/json'})
+        update = Update.get(nvr)
+        update.status = UpdateStatus.testing
+        update.request = None
+        update.pushed = True
+        update.autokarma = False
+        update.stable_karma = 1
+        update.test_gating_status = TestGatingStatus.passed
+        update.comment(self.db, 'works', 1, 'bowlofeggs')
+        self.db.commit()
+        self.app_settings['test_gating.required'] = True
+        app = TestApp(main({}, testing=u'guest', **self.app_settings))
+
+        resp = app.get('/updates/%s' % nvr, headers={'Accept': 'text/html'})
+
+        self.assertIn('Push to Batched', resp)
+        self.assertNotIn('Push to Stable', resp)
+        self.assertIn('Edit', resp)
+
     @unittest.skipIf(six.PY3, 'Not working with Python 3 yet')
     @mock.patch(**mock_valid_requirements)
     @mock.patch('bodhi.server.notifications.publish')
diff --git a/bodhi/tests/server/test_models.py b/bodhi/tests/server/test_models.py
--- a/bodhi/tests/server/test_models.py
+++ b/bodhi/tests/server/test_models.py
@@ -2834,6 +2834,24 @@ def test_check_requirements_koji_error(self, mock_multiCall,
         self.assertIn("Failed retrieving requirements results:", reason)
         self.assertIn("Error retrieving data from Koji for", reason)
 
+    def test_check_requirements_test_gating_status_failed(self):
+        """check_requirements() should return False when test_gating_status is failed."""
+        self.obj.requirements = ''
+        self.obj.test_gating_status = model.TestGatingStatus.failed
+
+        with mock.patch.dict(config, {'test_gating.required': True}):
+            self.assertEqual(self.obj.check_requirements(self.db, config),
+                             (False, 'Required tests did not pass on this update'))
+
+    def test_check_requirements_test_gating_status_passed(self):
+        """check_requirements() should return True when test_gating_status is passed."""
+        self.obj.requirements = ''
+        self.obj.test_gating_status = model.TestGatingStatus.passed
+
+        with mock.patch.dict(config, {'test_gating.required': True}):
+            self.assertEqual(self.obj.check_requirements(self.db, config),
+                             (True, 'No checks required.'))
+
     def test_test_cases_with_no_dupes(self):
         update = self.get_update(name=u"FullTestCasesWithNoDupes")
         package = update.builds[0].package