Repository: fedora-infra/bodhi

Issue:
Newlines in the greenwave_summary_string break the JavaScript that renders test results
While testing #2273 in staging today, I noticed that updates that having missing tests do not render anything in the test results tab. I was able to reproduce this locally and I found that the problem is that the newline gets included in the JavaScript in a way that makes the JS invalid (and the browser refuses to process the rest of the JS as a result). The issue is [here](https://github.com/fedora-infra/bodhi/blob/3.6.1/bodhi/server/templates/update.html#L86):

```
var summary = '${update.greenwave_summary_string}';
```

It may be best to either use a different string to split the missing tests from the summary (perhaps something more unique too, like :::), or to put it into a new field of its own (which might be cleaner but will require a database migration).

This is a 3.7.0 release blocker.


Discussion:
Right, there's no way to put a newline in a javascript string, other than an explicit `\n` (i.e. escaped).
As stated in #2334 I think I am going to add a new field to store the list of missing tests in. This way we don't need a special separator character.
Disclaimer: I'm real bad at UX stuff.

I've been fiddling today with getting Bodhi to give more useful information to the user since I'm giving Bodhi a dedicated field to store Greenwave's unsatisfied requirements. Rather than just giving the user a list of test names that didn't run, I think we should just give them the full list of unsatisfied requirements. It's a little tricky to massage the data from Greenwave into a human comprehensible format, but here's a stab I've taken at it:

![screenshot from 2018-05-01 14-56-01](https://user-images.githubusercontent.com/354506/39488433-45972684-4d50-11e8-9908-0e872fa69506.png)

Resolving patch:
diff --git a/bodhi/server/migrations/versions/59c0f5fbc1b2_add_a_greenwave_unsatisfied_.py b/bodhi/server/migrations/versions/59c0f5fbc1b2_add_a_greenwave_unsatisfied_.py
new file mode 100644
--- /dev/null
+++ b/bodhi/server/migrations/versions/59c0f5fbc1b2_add_a_greenwave_unsatisfied_.py
@@ -0,0 +1,42 @@
+# Copyright (c) 2018 Red Hat, Inc.
+#
+# This file is part of Bodhi.
+#
+# This program is free software; you can redistribute it and/or
+# modify it under the terms of the GNU General Public License
+# as published by the Free Software Foundation; either version 2
+# of the License, or (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
+"""
+Add a greenwave_unsatisfied_requirements column to the updates table.
+
+Revision ID: 59c0f5fbc1b2
+Revises: be25565a1211
+Create Date: 2018-05-01 15:37:07.346034
+"""
+from alembic import op
+import sqlalchemy as sa
+
+
+# revision identifiers, used by Alembic.
+revision = '59c0f5fbc1b2'
+down_revision = 'be25565a1211'
+
+
+def upgrade():
+    """Add a greenwave_unsatisfied_requirements to the updates table."""
+    op.add_column('updates',
+                  sa.Column('greenwave_unsatisfied_requirements', sa.UnicodeText(), nullable=True))
+
+
+def downgrade():
+    """Drop the greenwave_unsatisfied_requirements from the updates table."""
+    op.drop_column('updates', 'greenwave_unsatisfied_requirements')
diff --git a/bodhi/server/models.py b/bodhi/server/models.py
--- a/bodhi/server/models.py
+++ b/bodhi/server/models.py
@@ -1677,6 +1677,9 @@ class Update(Base):
             Greenwave integration was not enabled when the update was created.
         greenwave_summary_string (unicode): A short summary of the outcome from Greenwave
             (e.g. 2 of 32 required tests failed).
+        greenwave_unsatisfied_requirements (unicode): When test_gating_status is failed, Bodhi will
+            set this to a JSON representation of the unsatisfied_requirements field from Greewave's
+            response.
         compose (Compose): The :class:`Compose` that this update is currently being mashed in. The
             update is locked if this is defined.
     """
@@ -1755,6 +1758,7 @@ class Update(Base):
     # Greenwave
     test_gating_status = Column(TestGatingStatus.db_type(), default=None, nullable=True)
     greenwave_summary_string = Column(Unicode(255))
+    greenwave_unsatisfied_requirements = Column(UnicodeText, nullable=True)
 
     # WARNING: consumers/masher.py assumes that this validation is performed!
     @validates('builds')
@@ -1957,14 +1961,11 @@ def update_test_gating_status(self):
                 self.test_gating_status = TestGatingStatus.ignored
             else:
                 self.test_gating_status = TestGatingStatus.passed
+            self.greenwave_unsatisfied_requirements = None
         else:
             self.test_gating_status = TestGatingStatus.failed
-            missing_reqs = [
-                req['testcase'] for req in
-                decision.get('unsatisfied_requirements', [])
-            ]
-            if missing_reqs:
-                decision['summary'] += '\n Missing: %s' % (', '.join(missing_reqs))
+            self.greenwave_unsatisfied_requirements = json.dumps(
+                decision.get('unsatisfied_requirements', []))
         self.greenwave_summary_string = decision['summary']
 
     @classmethod
diff --git a/bodhi/server/util.py b/bodhi/server/util.py
--- a/bodhi/server/util.py
+++ b/bodhi/server/util.py
@@ -552,8 +552,48 @@ def test_gating_status2html(context, status, reason=None):
     html = u'<span class="label label-%s">Tests %s</span>' % (label_class, description)
     from bodhi.server.models import TestGatingStatus
     if status not in [None, TestGatingStatus.ignored, TestGatingStatus.passed] and reason:
-        html += u'<p><a class="gating-summary" href="#automatedtests">%s</a></p>' % (
-                reason.split('\n', 1)[0])
+        html += u'<p><a class="gating-summary" href="#automatedtests">%s</a></p>' % reason
+    return html
+
+
+def greenwave_unsatisfied_requirements_html(context, update):
+    """
+    Return an html representation of the given update's greenwave_unsatisfied_requirements field.
+
+    This function collates the unsatisfied requirements into a heirarchy of requirement type, then
+    test name, and then a list of builds that fail to satisfy that requirement type for that test
+    name.
+
+    Args:
+        context (mako.runtime.Context): Unused.
+        update (bodhi.server.models.Update): The update to render unsatisfied requirements for.
+    Returns:
+        str: A human readable representation of the update's unsatisfied requirements.
+    """
+    if not update.greenwave_unsatisfied_requirements:
+        return ''
+
+    # The Greenwave data is in a list of dictionaries that describe unsatisfied requirements. Let's
+    # collate that data into type of requirement (such as missing test, failed test, etc.) --> test
+    # name --> list of nvrs that fail that test name. To do this, we'll build a dictionary of
+    # dictionaries of lists, like this:
+    #
+    # {'test-result-missing': {'dist.rpmdeplint': ['bodhi-3.6.1-fc28']}}
+    reqs = defaultdict(lambda: defaultdict(list))
+    for r in json.loads(update.greenwave_unsatisfied_requirements):
+        # The Greenwave response contains a lot of duplicate information. For now, let's ignore
+        # anything that isn't a koji build. https://pagure.io/greenwave/issue/169
+        if r['item'].get('type') == 'koji_build':
+            reqs[r['type']][r['testcase']].append(r['item']['item'])
+    # Now that we've built our data structure, let's render it into an HTML snippet that can be
+    # included in the update page.
+    html = '<div>Unsatisfied requirements:'
+    for t in reqs.keys():
+        html = html + '<div>{}: {}</div>'.format(
+            t,
+            ', '.join([' {} ({})'.format(testcase, ', '.join(reqs[t][testcase]))
+                      for testcase in reqs[t].keys()]))
+    html = html + '</div>'
     return html
 
 


Test patch:
diff --git a/bodhi/tests/server/consumers/test_updates.py b/bodhi/tests/server/consumers/test_updates.py
--- a/bodhi/tests/server/consumers/test_updates.py
+++ b/bodhi/tests/server/consumers/test_updates.py
@@ -19,6 +19,7 @@
 """This test suite contains tests for the bodhi.server.consumers.updates module."""
 
 import copy
+import json
 import unittest
 
 import mock
@@ -108,8 +109,16 @@ def test_gating_required_false(self):
                 'policies_satisfied': False,
                 'summary': u'what have you done‽',
                 'applicable_policies': ['taskotron_release_critical_tasks'],
-                'unsatisfied_requirements': ['some arbitrary test you disagree with']
-            }
+                'unsatisfied_requirements': [
+                    {u'testcase': u'dist.rpmdeplint',
+                     u'item': {u'item': u'bodhi-2.0-1.fc17', u'type': u'koji_build'},
+                     u'type': u'test-result-missing', u'scenario': None},
+                    {u'testcase': u'dist.rpmdeplint',
+                     u'item': {u'original_spec_nvr': u'bodhi-2.0-1.fc17'},
+                     u'type': u'test-result-missing', u'scenario': None},
+                    {u'testcase': u'dist.rpmdeplint',
+                     u'item': {u'item': update.alias, u'type': u'bodhi_update'},
+                     u'type': u'test-result-missing', u'scenario': None}]}
             mock_greenwave.return_value = greenwave_response
 
             h.consume(message)
@@ -117,6 +126,7 @@ def test_gating_required_false(self):
         update = models.Update.query.filter_by(title=u'bodhi-2.0-1.fc17').one()
         self.assertIsNone(update.test_gating_status)
         self.assertIsNone(update.greenwave_summary_string)
+        self.assertIsNone(update.greenwave_unsatisfied_requirements)
 
     @mock.patch.dict('bodhi.server.config.config', {'test_gating.required': True})
     def test_gating_required_true(self):
@@ -139,17 +149,25 @@ def test_gating_required_true(self):
                 'policies_satisfied': False,
                 'summary': u'what have you done‽',
                 'applicable_policies': ['taskotron_release_critical_tasks'],
-                'unsatisfied_requirements': [{'testcase': 'some arbitrary test you disagree with'}]
-            }
+                'unsatisfied_requirements': [
+                    {u'testcase': u'dist.rpmdeplint',
+                     u'item': {u'item': u'bodhi-2.0-1.fc17', u'type': u'koji_build'},
+                     u'type': u'test-result-missing', u'scenario': None},
+                    {u'testcase': u'dist.rpmdeplint',
+                     u'item': {u'original_spec_nvr': u'bodhi-2.0-1.fc17'},
+                     u'type': u'test-result-missing', u'scenario': None},
+                    {u'testcase': u'dist.rpmdeplint',
+                     u'item': {u'item': update.alias, u'type': u'bodhi_update'},
+                     u'type': u'test-result-missing', u'scenario': None}]}
             mock_greenwave.return_value = greenwave_response
 
             h.consume(message)
 
         update = models.Update.query.filter_by(title=u'bodhi-2.0-1.fc17').one()
         self.assertEqual(update.test_gating_status, models.TestGatingStatus.failed)
-        self.assertEqual(
-            update.greenwave_summary_string,
-            u'what have you done‽\n Missing: some arbitrary test you disagree with')
+        self.assertEqual(update.greenwave_summary_string, u'what have you done‽')
+        self.assertEqual(update.greenwave_unsatisfied_requirements,
+                         json.dumps(greenwave_response['unsatisfied_requirements']))
 
     # We're going to use side effects to mock but still call work_on_bugs and fetch_test_cases so we
     # can ensure that we aren't raising Exceptions from them, while allowing us to only assert that
diff --git a/bodhi/tests/server/scripts/test_check_policies.py b/bodhi/tests/server/scripts/test_check_policies.py
--- a/bodhi/tests/server/scripts/test_check_policies.py
+++ b/bodhi/tests/server/scripts/test_check_policies.py
@@ -1,5 +1,5 @@
 # -*- coding: utf-8 -*-
-# Copyright © 2017 Red Hat, Inc.
+# Copyright © 2017-2018 Red Hat, Inc.
 #
 # This file is part of Bodhi.
 #
@@ -18,6 +18,7 @@
 # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
 """This module contains tests for the bodhi.server.scripts.check_policies module."""
 import datetime
+import json
 
 from click import testing
 from mock import patch
@@ -50,6 +51,7 @@ def test_policies_satisfied(self):
             update = self.db.query(models.Update).filter(models.Update.id == update.id).one()
             self.assertEqual(update.test_gating_status, models.TestGatingStatus.passed)
             self.assertEqual(update.greenwave_summary_string, 'All tests passed')
+            self.assertIsNone(update.greenwave_unsatisfied_requirements)
 
         expected_query = {
             'product_version': 'fedora-17', 'decision_context': 'bodhi_update_push_stable',
@@ -82,6 +84,7 @@ def test_policies_pending_satisfied(self):
             update = self.db.query(models.Update).filter(models.Update.id == update.id).one()
             self.assertEqual(update.test_gating_status, models.TestGatingStatus.passed)
             self.assertEqual(update.greenwave_summary_string, 'All tests passed')
+            self.assertIsNone(update.greenwave_unsatisfied_requirements)
 
         expected_query = {
             'product_version': 'fedora-17', 'decision_context': 'bodhi_update_push_testing',
@@ -105,21 +108,24 @@ def test_policies_unsatisfied(self):
                 'policies_satisfied': False,
                 'summary': '1 of 2 tests are failed',
                 'applicable_policies': ['taskotron_release_critical_tasks'],
-                'unsatisfied_requirements': [{
-                    'item': "glibc-1.0-1.f26",
-                    'result_id': "123",
-                    'testcase': 'dist.depcheck',
-                    'type': 'test-result-failed'
-                }]
-            }
+                'unsatisfied_requirements': [
+                    {u'testcase': u'dist.rpmdeplint',
+                     u'item': {u'item': u'glibc-1.0-1.f26', u'type': u'koji_build'},
+                     u'type': u'test-result-missing', u'scenario': None},
+                    {u'testcase': u'dist.rpmdeplint',
+                     u'item': {u'original_spec_nvr': u'glibc-1.0-1.f26'},
+                     u'type': u'test-result-missing', u'scenario': None},
+                    {u'testcase': u'dist.rpmdeplint',
+                     u'item': {u'item': update.alias, u'type': u'bodhi_update'},
+                     u'type': u'test-result-missing', u'scenario': None}]}
             mock_greenwave.return_value = greenwave_response
             result = runner.invoke(check_policies.check, [])
             self.assertEqual(result.exit_code, 0)
             update = self.db.query(models.Update).filter(models.Update.id == update.id).one()
             self.assertEqual(update.test_gating_status, models.TestGatingStatus.failed)
-            self.assertEqual(
-                update.greenwave_summary_string,
-                '1 of 2 tests are failed\n Missing: dist.depcheck')
+            self.assertEqual(update.greenwave_summary_string, '1 of 2 tests are failed')
+            self.assertEqual(update.greenwave_unsatisfied_requirements,
+                             json.dumps(greenwave_response['unsatisfied_requirements']))
 
         expected_query = {
             'product_version': 'fedora-17', 'decision_context': 'bodhi_update_push_stable',
diff --git a/bodhi/tests/server/test_models.py b/bodhi/tests/server/test_models.py
--- a/bodhi/tests/server/test_models.py
+++ b/bodhi/tests/server/test_models.py
@@ -2991,7 +2991,7 @@ def test_can_waive_test_results_of_an_update(self, mock_waiverdb, mock_greenwave
             "applicable_policies": ["1"],
             "unsatisfied_requirements": [
                 {
-                    'item': {"item": "%s" % update.alias, "type": "koji_build"},
+                    'item': {"item": "%s" % update.builds[0].nvr, "type": "koji_build"},
                     'result_id': "123",
                     'testcase': 'dist.depcheck',
                     'type': 'test-result-failed'
@@ -3004,7 +3004,7 @@ def test_can_waive_test_results_of_an_update(self, mock_waiverdb, mock_greenwave
                               'waiverdb.access_token': 'abc'}):
             update.waive_test_results('foo', 'this is not true!')
             wdata = {
-                'subject': {"item": "%s" % update.alias, "type": "koji_build"},
+                'subject': {"item": "%s" % update.builds[0].nvr, "type": "koji_build"},
                 'testcase': 'dist.depcheck',
                 'product_version': update.product_version,
                 'waived': True,
diff --git a/bodhi/tests/server/test_util.py b/bodhi/tests/server/test_util.py
--- a/bodhi/tests/server/test_util.py
+++ b/bodhi/tests/server/test_util.py
@@ -17,6 +17,7 @@
 # along with this program; if not, write to the Free Software
 # Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
 import gzip
+import json
 import os
 import shutil
 import subprocess
@@ -210,6 +211,55 @@ def json(self):
         sleep.assert_called_once_with(1)
 
 
+class TestGreenwaveUnsatisfiedRequirementsHTML(base.BaseTestCase):
+    """Test the greenwave_unsatisfied_requirements_html() function."""
+
+    def test_no_unsatisfied_requirements(self):
+        u = Update.query.first()
+        u.greenwave_unsatisfied_requirements = None
+
+        self.assertEqual(util.greenwave_unsatisfied_requirements_html(None, u), '')
+
+    def test_unsatisfied_requirements(self):
+        u = Update.query.first()
+        u.greenwave_unsatisfied_requirements = json.dumps([
+            {'testcase': 'dist.rpmdeplint',
+             'item': {'item': 'python-rpdb-0.1.6-7.fc27', 'type': 'koji_build'},
+             'type': 'test-result-missing', 'scenario': None},
+            {'testcase': 'dist.rpmdeplint',
+             'item': {'original_spec_nvr': 'python-rpdb-0.1.6-7.fc27'},
+             'type': 'test-result-missing', 'scenario': None},
+            {'testcase': 'dist.rpmdeplint',
+             'item': {'item': 'FEDORA-2018-6b448bbc48', 'type': 'bodhi_update'},
+             'type': 'test-result-missing', 'scenario': None},
+            {'testcase': 'dist.rpmdeplint',
+             'item': {'item': 'bodhi-3.6.1-1.fc28', 'type': 'koji_build'},
+             'type': 'test-result-missing', 'scenario': None},
+            {'testcase': 'dist.someothertest',
+             'item': {'item': 'bodhi-3.6.1-1.fc28', 'type': 'koji_build'},
+             'type': 'test-result-sucks', 'scenario': None},
+            {'testcase': 'dist.anothertest',
+             'item': {'item': 'python-rpdb-0.1.6-7.fc27', 'type': 'koji_build'},
+             'type': 'test-result-missing', 'scenario': None}])
+
+        html = util.greenwave_unsatisfied_requirements_html(None, u)
+
+        # The bodhi_update type gets ignored in the HTML for now (we might change this later).
+        # We can't assert the order of the divs in the html because they are different between
+        # Python 2 and 3. We also don't really care what order they appear in anyway, so let's just
+        # make sure they appear in the resulting HTML.
+        self.assertTrue(html.startswith('<div>Unsatisfied requirements:'))
+        self.assertIn(
+            ('<div>test-result-missing:  '
+             'dist.rpmdeplint (python-rpdb-0.1.6-7.fc27, bodhi-3.6.1-1.fc28),  '
+             'dist.anothertest (python-rpdb-0.1.6-7.fc27)</div>'),
+            html)
+        self.assertIn(
+            '<div>test-result-sucks:  dist.someothertest (bodhi-3.6.1-1.fc28)</div>', html)
+        # Let's make sure we close things out correctly.
+        self.assertTrue(html.endswith('</div></div>'))
+
+
 class TestKarma2HTML(unittest.TestCase):
     """Test the karma2html() function."""