Repository: fedora-infra/bodhi

Issue:
The composer Nacks messages when they reference composes that no longer exist
If messages get into Bodhi's queue that reference Compose objects that no longer exist in the database, the Composer raises an Exception. This causes the message to be Nack'd, which causes it to go back into the queue and Nack loop forever. Eventually this will cause the queue to be filled with inactionable messages and will prevent actionable messages from getting through.

```
2019-06-17 15:09:32,761 ERROR [bodhi][PoolThread-twisted.internet.reactor-0] No row was found for one(): Unable to handle message: Id: cf8dd3eb-ec01-4fbb-9b4f-b314a1fa2ef9
Topic: org.fedoraproject.prod.bodhi.composer.start
Headers: {
    "fedora_messaging_schema": "bodhi.composer.start.v1",
    "fedora_messaging_severity": 20,
    "fedora_messaging_user_releng": true,
    "sent-at": "2019-06-16T00:00:11+00:00"
}
Body: {
    "agent": "releng",
    "api_version": 2,
    "composes": [
        {
            "content_type": "rpm",
            "release_id": 23,
            "request": "testing",
            "security": true
        },
        {
            "content_type": "rpm",
            "release_id": 28,
            "request": "testing",
            "security": true
        },
        {
            "content_type": "rpm",
            "release_id": 8,
            "request": "stable",
            "security": false
        },
        {
            "content_type": "rpm",
            "release_id": 28,
            "request": "stable",
            "security": false
        },
        {
            "content_type": "rpm",
            "release_id": 23,
            "request": "stable",
            "security": false
        },
        {
            "content_type": "rpm",
            "release_id": 10,
            "request": "testing",
            "security": false
        },
        {
            "content_type": "rpm",
            "release_id": 8,
            "request": "testing",
            "security": false
        },
        {
            "content_type": "module",
            "release_id": 29,
            "request": "testing",
            "security": false
        },
        {
            "content_type": "module",
            "release_id": 24,
            "request": "testing",
            "security": false
        }
    ],
    "resume": false
}
Traceback (most recent call last):
  File "/usr/lib/python3.7/site-packages/bodhi/server/consumers/__init__.py", line 77, in __call__
    self.composer_handler(msg)
  File "/usr/lib/python3.7/site-packages/bodhi/server/consumers/composer.py", line 192, in __call__
    for compose in self._get_composes(message):
  File "/usr/lib/python3.7/site-packages/bodhi/server/consumers/composer.py", line 236, in _get_composes
    composes = [Compose.from_dict(db, c) for c in msg['composes']]
  File "/usr/lib/python3.7/site-packages/bodhi/server/consumers/composer.py", line 236, in <listcomp>
    composes = [Compose.from_dict(db, c) for c in msg['composes']]
  File "/usr/lib/python3.7/site-packages/bodhi/server/models.py", line 3607, in from_dict
    request=UpdateRequest.from_string(compose['request'])).one()
  File "/usr/lib64/python3.7/site-packages/sqlalchemy/orm/query.py", line 3046, in one
    raise orm_exc.NoResultFound("No row was found for one()")
sqlalchemy.orm.exc.NoResultFound: No row was found for one()
```


Discussion:

Resolving patch:
diff --git a/bodhi/server/consumers/composer.py b/bodhi/server/consumers/composer.py
--- a/bodhi/server/consumers/composer.py
+++ b/bodhi/server/consumers/composer.py
@@ -41,6 +41,7 @@
 
 import jinja2
 import fedora_messaging
+import sqlalchemy.orm.exc
 
 from bodhi.messages.schemas import compose as compose_schemas, update as update_schemas
 from bodhi.server import buildsys, notifications, mail
@@ -233,7 +234,17 @@ def _get_composes(self, msg: dict):
         """
         with self.db_factory() as db:
             if 'api_version' in msg and msg['api_version'] == 2:
-                composes = [Compose.from_dict(db, c) for c in msg['composes']]
+                try:
+                    composes = [Compose.from_dict(db, c) for c in msg['composes']]
+                except sqlalchemy.orm.exc.NoResultFound:
+                    # It is possible for messages to get into our queue that reference Composes that
+                    # no longer exist. If this happens, we really just want to ignore the message so
+                    # that it gets dropped. In particular, we do not want to raise an Exception when
+                    # this happens, because that will Nack the message put it back into the queue,
+                    # resulting in a Nack loop.
+                    # See https://github.com/fedora-infra/bodhi/issues/3318
+                    log.info('Ignoring a compose message that references non-existing Composes')
+                    return []
             else:
                 raise ValueError('Unable to process message: {}'.format(msg))
 


Test patch:
diff --git a/bodhi/tests/server/consumers/test_composer.py b/bodhi/tests/server/consumers/test_composer.py
--- a/bodhi/tests/server/consumers/test_composer.py
+++ b/bodhi/tests/server/consumers/test_composer.py
@@ -331,15 +331,23 @@ def test__get_composes_api_3(self):
             # The Compose's state should not have been altered.
             self.assertEqual(compose.state, ComposeState.requested)
 
-    def test_push_invalid_compose(self):
-        msg = self._make_msg()
-        msg.body['composes'][0]['release_id'] = 65535
+    @mock.patch('bodhi.server.consumers.composer.log.info')
+    def test__get_composes_not_found(self, info):
+        """
+        Test _get_composes() with a message referencing a Compose that does not exist.
 
-        with self.assertRaises(Exception) as exc:
-            with mock_sends(*[base_schemas.BodhiMessage] * 5):
-                self.handler(msg)
+        We don't want Bodhi to Nack messages that reference Composes that no longer exist, because
+        that will lead to Nack loops. If we receive a message like this, _get_composes() should just
+        return the empty list. See https://github.com/fedora-infra/bodhi/issues/3318
+        """
+        message = self._make_msg().body
+        message['composes'][0]['release_id'] = 65535
+
+        composes = self.handler._get_composes(message)
 
-        self.assertEqual(str(exc.exception), 'No row was found for one()')
+        self.assertEqual(composes, [])
+        info.assert_called_once_with(
+            'Ignoring a compose message that references non-existing Composes')
 
     def test__get_composes_duplicate(self):
         """Test _get_composes() when a duplicate message is received."""