Repository: fedora-infra/bodhi

Issue:
Bodhi does not use the WaiverDB API correctly
We gave Bodhi an auth token for WaiverDB today and it turned out that Bodhi's code for using WaiverDB's API is incorrect in two ways:

0. It uses "proxy_user" instead of "username" to specify the acting user.
1. It sends all waivers in one API call, but WaiverDB needs one waiver per API call.


Discussion:

Resolving patch:
diff --git a/bodhi/server/models.py b/bodhi/server/models.py
--- a/bodhi/server/models.py
+++ b/bodhi/server/models.py
@@ -1959,6 +1959,12 @@ def update_test_gating_status(self):
                 self.test_gating_status = TestGatingStatus.passed
         else:
             self.test_gating_status = TestGatingStatus.failed
+            missing_reqs = [
+                req['testcase'] for req in
+                decision.get('unsatisfied_requirements', [])
+            ]
+            if missing_reqs:
+                decision['summary'] += '\n Missing: %s' % (', '.join(missing_reqs))
         self.greenwave_summary_string = decision['summary']
 
     @classmethod
@@ -2624,17 +2630,17 @@ def waive_test_results(self, username, comment=None):
             'subject': self.greenwave_subject
         }
         decision = greenwave_api_post('{}/decision'.format(config.get('greenwave_api_url')), data)
-        results = [dict(subject=req['item'], testcase=req['testcase']) for req in
-                   decision['unsatisfied_requirements']]
-        log.debug('Waiving test results: %s' % results)
-        data = {
-            'results': results,
-            'product_version': self.product_version,
-            'waived': True,
-            'proxy_user': username,
-            'comment': comment
-        }
-        waiverdb_api_post('{}/waivers/'.format(config.get('waiverdb_api_url')), data)
+        for requirement in decision['unsatisfied_requirements']:
+            data = {
+                'subject': requirement['item'],
+                'testcase': requirement['testcase'],
+                'product_version': self.product_version,
+                'waived': True,
+                'username': username,
+                'comment': comment
+            }
+            log.debug('Waiving test results: %s' % data)
+            waiverdb_api_post('{}/waivers/'.format(config.get('waiverdb_api_url')), data)
 
     def add_tag(self, tag):
         """
diff --git a/bodhi/server/scripts/check_policies.py b/bodhi/server/scripts/check_policies.py
--- a/bodhi/server/scripts/check_policies.py
+++ b/bodhi/server/scripts/check_policies.py
@@ -35,9 +35,20 @@ def check():
     initialize_db(config.config)
     session = Session()
 
-    updates = models.Update.query.filter(models.Update.status.in_(
-        [models.UpdateStatus.pending, models.UpdateStatus.testing])).order_by(
-        models.Update.id.desc())
+    updates = models.Update.query.filter(
+        models.Update.status.in_(
+            [models.UpdateStatus.pending, models.UpdateStatus.testing])
+    ).filter(
+        models.Update.release_id == models.Release.id
+    ).filter(
+        models.Release.state.in_(
+            [models.ReleaseState.current, models.ReleaseState.pending])
+    ).order_by(
+        # Check the older updates first so there is more time for the newer to
+        # get their test results
+        models.Update.id.asc()
+    )
+
     for update in updates:
         try:
             update.update_test_gating_status()
diff --git a/bodhi/server/util.py b/bodhi/server/util.py
--- a/bodhi/server/util.py
+++ b/bodhi/server/util.py
@@ -552,7 +552,8 @@ def test_gating_status2html(context, status, reason=None):
     html = u'<span class="label label-%s">Tests %s</span>' % (label_class, description)
     from bodhi.server.models import TestGatingStatus
     if status not in [None, TestGatingStatus.ignored, TestGatingStatus.passed] and reason:
-        html += u'<p><a class="gating-summary" href="#automatedtests">%s</a></p>' % reason
+        html += u'<p><a class="gating-summary" href="#automatedtests">%s</a></p>' % (
+                reason.split('\n', 1)[0])
     return html
 
 


Test patch:
diff --git a/bodhi/tests/server/consumers/test_updates.py b/bodhi/tests/server/consumers/test_updates.py
--- a/bodhi/tests/server/consumers/test_updates.py
+++ b/bodhi/tests/server/consumers/test_updates.py
@@ -139,7 +139,7 @@ def test_gating_required_true(self):
                 'policies_satisfied': False,
                 'summary': u'what have you done‽',
                 'applicable_policies': ['taskotron_release_critical_tasks'],
-                'unsatisfied_requirements': ['some arbitrary test you disagree with']
+                'unsatisfied_requirements': [{'testcase': 'some arbitrary test you disagree with'}]
             }
             mock_greenwave.return_value = greenwave_response
 
@@ -147,7 +147,9 @@ def test_gating_required_true(self):
 
         update = models.Update.query.filter_by(title=u'bodhi-2.0-1.fc17').one()
         self.assertEqual(update.test_gating_status, models.TestGatingStatus.failed)
-        self.assertEqual(update.greenwave_summary_string, u'what have you done‽')
+        self.assertEqual(
+            update.greenwave_summary_string,
+            u'what have you done‽\n Missing: some arbitrary test you disagree with')
 
     # We're going to use side effects to mock but still call work_on_bugs and fetch_test_cases so we
     # can ensure that we aren't raising Exceptions from them, while allowing us to only assert that
diff --git a/bodhi/tests/server/scripts/test_check_policies.py b/bodhi/tests/server/scripts/test_check_policies.py
--- a/bodhi/tests/server/scripts/test_check_policies.py
+++ b/bodhi/tests/server/scripts/test_check_policies.py
@@ -117,7 +117,9 @@ def test_policies_unsatisfied(self):
             self.assertEqual(result.exit_code, 0)
             update = self.db.query(models.Update).filter(models.Update.id == update.id).one()
             self.assertEqual(update.test_gating_status, models.TestGatingStatus.failed)
-            self.assertEqual(update.greenwave_summary_string, '1 of 2 tests are failed')
+            self.assertEqual(
+                update.greenwave_summary_string,
+                '1 of 2 tests are failed\n Missing: dist.depcheck')
 
         expected_query = {
             'product_version': 'fedora-17', 'decision_context': 'bodhi_update_push_stable',
@@ -220,3 +222,25 @@ def test_unrestricted_policy(self):
                  'type': 'bodhi_update'}]}
         mock_greenwave.assert_called_once_with(config['greenwave_api_url'] + '/decision',
                                                expected_query)
+
+    @patch.dict(config, [('greenwave_api_url', 'http://domain.local')])
+    def test_archived_release_updates(self):
+        """Assert that updates for archived releases isn't being considered
+        by the script.
+        """
+        # Archive the F17 release
+        rel = self.db.query(models.Release).filter_by(name='F17').one()
+        rel.state = models.ReleaseState.archived
+        self.db.commit()
+
+        runner = testing.CliRunner()
+        update = self.db.query(models.Update).all()[0]
+        update.status = models.UpdateStatus.testing
+        self.db.commit()
+        with patch('bodhi.server.models.util.greenwave_api_post') as mock_greenwave:
+            mock_greenwave.side_effect = Exception(
+                'Greenwave should not be accessed for archived releases.')
+            result = runner.invoke(check_policies.check, [])
+            self.assertEqual(result.exit_code, 0)
+
+        self.assertEqual(mock_greenwave.call_count, 0)
diff --git a/bodhi/tests/server/test_models.py b/bodhi/tests/server/test_models.py
--- a/bodhi/tests/server/test_models.py
+++ b/bodhi/tests/server/test_models.py
@@ -2926,6 +2926,59 @@ def test_cannot_waive_test_results_of_an_update_which_is_locked(self):
             str(exc.exception),
             ("Can't waive test results on a locked update"))
 
+    @mock.patch('bodhi.server.models.greenwave_api_post')
+    @mock.patch('bodhi.server.util.http_session.post')
+    def test_can_waive_multiple_test_results_of_an_update(self, post, greenwave_api_post):
+        """Multiple failed tests getting waived should cause multiple calls to waiverdb."""
+        self.obj.status = UpdateStatus.testing
+        self.obj.test_gating_status = TestGatingStatus.failed
+        greenwave_api_post.return_value = {
+            "policies_satisfied": False,
+            "summary": "3 of 15 required tests failed",
+            "applicable_policies": ["1"],
+            "unsatisfied_requirements": [
+                {'item': {"item": "bodhi-3.6.0-1.fc28", "type": "koji_build"}, 'result_id': "123",
+                 'testcase': 'dist.depcheck', 'type': 'test-result-failed'},
+                {'item': {"item": "bodhi-3.6.0-1.fc28", "type": "koji_build"}, 'result_id': "124",
+                 'testcase': 'dist.rpmdeplint', 'type': 'test-result-failed'},
+                {'item': {"item": "bodhi-3.6.0-1.fc28", "type": "koji_build"}, 'result_id': "125",
+                 'testcase': 'dist.someothertest', 'type': 'test-result-failed'}]}
+        post.return_value.status_code = 200
+
+        with mock.patch.dict('bodhi.server.config.config',
+                             {'test_gating.required': True,
+                              'waiverdb.access_token': 'abc'}):
+            self.obj.waive_test_results('foo', 'this is not true!')
+
+        expected_calls = []
+        for test in ('dist.depcheck', 'dist.rpmdeplint', 'dist.someothertest'):
+            data = {
+                "username": "foo", "comment": "this is not true!", "waived": True,
+                "product_version": "{}".format(self.obj.product_version),
+                "testcase": "{}".format(test),
+                "subject": {"item": "bodhi-3.6.0-1.fc28", "type": "koji_build"}}
+            expected_calls.append(mock.call(
+                '{}/waivers/'.format(config.get('waiverdb_api_url')),
+                data=json.dumps(data),
+                headers={'Content-Type': 'application/json', 'Authorization': 'Bearer abc'},
+                timeout=60))
+            expected_calls.append(mock.call().json())
+        for i, v in enumerate(expected_calls):
+            # The even numbered calls have a JSON serialized data string in them, and the order of
+            # the keys is not guaranteed to be the same by Python. For these, we will just make sure
+            # that the interpreted JSON is equal rather than verifying that the strings are equal.
+            if not i % 2:
+                self.assertEqual(post.mock_calls[i][1], expected_calls[i][1])
+                self.assertEqual(post.mock_calls[i][2].keys(), v[2].keys())
+                for k in v[2].keys():
+                    if k == 'data':
+                        self.assertEqual(
+                            json.loads(post.mock_calls[i][2]['data']), json.loads(v[2]['data']))
+                    else:
+                        self.assertEqual(post.mock_calls[i][2][k], expected_calls[i][2][k])
+            else:
+                self.assertEqual(post.mock_calls[i], v)
+
     @mock.patch('bodhi.server.models.greenwave_api_post')
     @mock.patch('bodhi.server.models.waiverdb_api_post')
     def test_can_waive_test_results_of_an_update(self, mock_waiverdb, mock_greenwave):
@@ -2951,11 +3004,11 @@ def test_can_waive_test_results_of_an_update(self, mock_waiverdb, mock_greenwave
                               'waiverdb.access_token': 'abc'}):
             update.waive_test_results('foo', 'this is not true!')
             wdata = {
-                'results': [{'subject': {"item": "%s" % update.alias, "type": "koji_build"},
-                             'testcase': 'dist.depcheck'}],
+                'subject': {"item": "%s" % update.alias, "type": "koji_build"},
+                'testcase': 'dist.depcheck',
                 'product_version': update.product_version,
                 'waived': True,
-                'proxy_user': 'foo',
+                'username': 'foo',
                 'comment': 'this is not true!'
             }
             mock_waiverdb.assert_called_once_with(