Repository: fedora-infra/bodhi

Issue:
The compose fails if any updates are kicked out due to failing test gating
We started enforcing test gating in the composer in #2346. Today some of the composes failed after kicking some updates out of the compose for failing the test gating check, with this message:

Message
-------
[2018-05-09 14:35:37][bodhi.server   ERROR]
```python
Unable to determine requested tag for libnfs-1.11.0-1.fc27.
```


Process Details
---------------
- host:     bodhi-backend01.phx2.fedoraproject.org
- PID:      57207
- name:     fedmsg-hub
- command:  /usr/bin/python2 /usr/bin/fedmsg-hub
- msg_id:   

Callstack that lead to the logging statement
--------------------------------------------
```python
  File "/usr/lib64/python2.7/threading.py", line 777 in __bootstrap
    self.__bootstrap_inner()
  File "/usr/lib64/python2.7/threading.py", line 804 in __bootstrap_inner
    self.run()
  File "/usr/lib/python2.7/site-packages/bodhi/server/consumers/masher.py", line 373 in run
    self.work()
  File "/usr/lib/python2.7/site-packages/bodhi/server/consumers/masher.py", line 437 in work
    self.determine_and_perform_tag_actions()
  File "/usr/lib/python2.7/site-packages/bodhi/server/consumers/masher.py", line 69 in wrapper
    retval = method(self, *args, **kwargs)
  File "/usr/lib/python2.7/site-packages/bodhi/server/consumers/masher.py", line 589 in determine_and_perform_tag_actions
    self._determine_tag_actions()
  File "/usr/lib/python2.7/site-packages/bodhi/server/consumers/masher.py", line 621 in _determine_tag_actions
    move_tags.append((from_tag, update.requested_tag,
  File "/usr/lib/python2.7/site-packages/bodhi/server/models.py", line 3556 in requested_tag
    'Unable to determine requested tag for %s.' % self.title)
```


The compose later halted with this message:

Message
-------
[2018-05-09 14:35:39][fedmsg.consumers   ERROR]
```python
Exception in ComposerThread(f26-updates)
```


Process Details
---------------
- host:     bodhi-backend01.phx2.fedoraproject.org
- PID:      57207
- name:     fedmsg-hub
- command:  /usr/bin/python2 /usr/bin/fedmsg-hub
- msg_id:   

Callstack that lead to the logging statement
--------------------------------------------
```python
  File "/usr/lib64/python2.7/threading.py", line 777 in __bootstrap
    self.__bootstrap_inner()
  File "/usr/lib64/python2.7/threading.py", line 804 in __bootstrap_inner
    self.run()
  File "/usr/lib/python2.7/site-packages/bodhi/server/consumers/masher.py", line 373 in run
    self.work()
  File "/usr/lib/python2.7/site-packages/bodhi/server/consumers/masher.py", line 473 in work
    self.log.exception('Exception in ComposerThread(%s)' % self.id)
```
```
Traceback (most recent call last):
  File "/usr/lib/python2.7/site-packages/bodhi/server/consumers/masher.py", line 437, in work
    self.determine_and_perform_tag_actions()
  File "/usr/lib/python2.7/site-packages/bodhi/server/consumers/masher.py", line 69, in wrapper
    retval = method(self, *args, **kwargs)
  File "/usr/lib/python2.7/site-packages/bodhi/server/consumers/masher.py", line 590, in determine_and_perform_tag_actions
    self._perform_tag_actions()
  File "/usr/lib/python2.7/site-packages/bodhi/server/consumers/masher.py", line 652, in _perform_tag_actions
    failed_tasks = buildsys.wait_for_tasks([task[0] for task in results],
KeyError: 0
```

I believe this will happen relatively infrequently in the future, but could still happen if Greenwave changes its decision between the time the packager pushes an update to stable and the time the compose runs. Thus, we should fix it.


Discussion:
I am raising the priority on this as the severity is worse than I had realized when I wrote the ticket yesterday. I learned today that this traceback happened after several builds had their tags altered in Koji. When this traceback happened, those builds remained re-tagged in Koji but Bodhi did not record which ones had been altered. The next day when the compose was resumed, Bodhi ejected many updates from the compose, for example [this one](https://bodhi.fedoraproject.org/updates/FEDORA-2018-9dc3a15a3a):

> nodejs-6.14.2-1.fc26 ejected from the push because u"Cannot find relevant tag for nodejs-6.14.2-1.fc26. None of ['f26-updates', 'f26-updates-pending'] are in [u'f22-updates-testing', u'dist-6E-epel-testing', u'f21-updates-testing', u'f25-updates-testing', u'f24-updates-testing', u'epel7-testing', u'f27-modular-updates-testing', u'dist-5E-epel-testing', u'f23-updates-testing', u'f26-updates-testing', u'f28-updates-testing', u'f27-updates-testing', u'f28-modular-updates-testing']."

As of this morning, nodejs was tagged with f26-updates, but it was still considered "testing" by Bodhi.
I am working to fix the builds that have gotten into this state here:

https://pagure.io/fedora-infrastructure/issue/6925

Resolving patch:
diff --git a/bodhi/server/buildsys.py b/bodhi/server/buildsys.py
--- a/bodhi/server/buildsys.py
+++ b/bodhi/server/buildsys.py
@@ -172,16 +172,22 @@ def multiCall(self):
 
     def moveBuild(self, from_tag, to_tag, build, *args, **kw):
         """Emulate Koji's moveBuild."""
+        if to_tag is None:
+            raise RuntimeError('Attempt to tag {} with None.'.format(build))
         log.debug("moveBuild(%s, %s, %s)" % (from_tag, to_tag, build))
         DevBuildsys.__moved__.append((from_tag, to_tag, build))
 
     def tagBuild(self, tag, build, *args, **kw):
         """Emulate Koji's tagBuild."""
+        if tag is None:
+            raise RuntimeError('Attempt to tag {} with None.'.format(build))
         log.debug("tagBuild(%s, %s)" % (tag, build))
         DevBuildsys.__added__.append((tag, build))
 
     def untagBuild(self, tag, build, *args, **kw):
         """Emulate Koji's untagBuild."""
+        if tag is None:
+            raise RuntimeError('Attempt to untag {} with None.'.format(build))
         log.debug("untagBuild(%s, %s)" % (tag, build))
         DevBuildsys.__untag__.append((tag, build))
 
diff --git a/bodhi/server/consumers/masher.py b/bodhi/server/consumers/masher.py
--- a/bodhi/server/consumers/masher.py
+++ b/bodhi/server/consumers/masher.py
@@ -465,6 +465,10 @@ def perform_gating(self):
             if not result:
                 self.log.warn("%s failed gating: %s" % (update.title, reason))
                 self.eject_from_mash(update, reason)
+        # We may have removed some updates from this compose above, and do we don't want future
+        # reads on self.compose.updates to see those, so let's mark that attribute expired so
+        # sqlalchemy will requery for the composes instead of using its cached copy.
+        self.db.expire(self.compose, ['updates'])
 
     def eject_from_mash(self, update, reason):
         """
diff --git a/bodhi/server/models.py b/bodhi/server/models.py
--- a/bodhi/server/models.py
+++ b/bodhi/server/models.py
@@ -3556,8 +3556,9 @@ def requested_tag(self):
         Return the tag the update has requested.
 
         Returns:
-            basestring or None: The Koji tag that corresponds to the update's current request, or
-                None if the method is unable to do so.
+            basestring: The Koji tag that corresponds to the update's current request.
+        Raises:
+            RuntimeError: If a Koji tag is unable to be determined.
         """
         tag = None
         if self.request is UpdateRequest.stable:
@@ -3571,7 +3572,7 @@ def requested_tag(self):
         elif self.request is UpdateRequest.obsolete:
             tag = self.release.candidate_tag
         if not tag:
-            log.error(
+            raise RuntimeError(
                 'Unable to determine requested tag for %s.' % self.title)
         return tag
 


Test patch:
diff --git a/bodhi/tests/server/consumers/test_masher.py b/bodhi/tests/server/consumers/test_masher.py
--- a/bodhi/tests/server/consumers/test_masher.py
+++ b/bodhi/tests/server/consumers/test_masher.py
@@ -2200,6 +2200,26 @@ def test_primary_arches_undefined(self):
         self.assert_sems(0)
 
 
+class TestComposerThread_perform_gating(ComposerThreadBaseTestCase):
+    """Test the ComposerThread.perform_gating() method."""
+
+    def test_expires_compose_updates(self):
+        """Ensure that the method expires the compose's updates attribute."""
+        msg = self._make_msg()
+        t = ComposerThread(self.semmock, msg['body']['msg']['composes'][0],
+                           'bowlofeggs', log, self.Session, self.tempdir)
+        t.compose = Compose.from_dict(self.db, msg['body']['msg']['composes'][0])
+        t.compose.updates[0].test_gating_status = TestGatingStatus.failed
+        t.db = self.db
+        t.id = getattr(self.db.query(Release).one(), '{}_tag'.format('stable'))
+
+        t.perform_gating()
+
+        # Without the call to self.db.expire() at the end of perform_gating(), there would be 1
+        # update here.
+        self.assertEqual(len(t.compose.updates), 0)
+
+
 class TestComposerThread__perform_tag_actions(ComposerThreadBaseTestCase):
     """This test class contains tests for the ComposerThread._perform_tag_actions() method."""
     @mock.patch('bodhi.server.consumers.masher.buildsys.wait_for_tasks')
diff --git a/bodhi/tests/server/test_models.py b/bodhi/tests/server/test_models.py
--- a/bodhi/tests/server/test_models.py
+++ b/bodhi/tests/server/test_models.py
@@ -1721,6 +1721,16 @@ def test_days_to_stable_positive(self):
         self.assertEqual(update.mandatory_days_in_testing > update.days_in_testing, True)
         self.assertEqual(update.days_to_stable, 4)
 
+    def test_requested_tag_request_none(self):
+        """requested_tag() should raise RuntimeError if the Update's request is None."""
+        self.obj.request = None
+
+        with self.assertRaises(RuntimeError) as exc:
+            self.obj.requested_tag
+
+        self.assertEqual(str(exc.exception),
+                         'Unable to determine requested tag for {}.'.format(self.obj.title))
+
     def test_side_tag_locked_false(self):
         """Test the side_tag_locked property when it is false."""
         self.obj.status = model.UpdateStatus.side_tag_active