Repository: fedora-infra/bodhi

Issue:
bodhi-dequqe-stable dies if any update in the queue is no longer eligible to go stable
QuLogic from Freenode reported today that batched updates didn't go stable at 03:00 UTC like they should have. I confirmed that the cron job ran, but I didn't see any notes about its output. I then ran the command by hand and received this output:

```
[bowlofeggs@bodhi-backend01 ~][PROD]$ sudo -u apache /usr/bin/bodhi-dequeue-stable
No handlers could be found for logger "bodhi.server"
This update has not yet met the minimum testing requirements defined in the <a href="https://fedoraproject.org/wiki/Package_update_acceptance_criteria">Package Update Acceptance Criteria</a>
```

The [```dequeue_stable()```](https://github.com/fedora-infra/bodhi/blob/3.0.0/bodhi/server/scripts/dequeue_stable.py#L28-L46) function runs a large transaction with only a single try/except. It seems that some update in the queue no longer meets testing requirements (probably due to receiving a -1 karma after going to batched) and is raising an Exception when the tool attempts to mark it for stable. Since there is only one try/except handler, this causes the whole transaction to be rolled back.

It should be easy to fix this - we just need a try/except around each update.

Thanks to QuLogic from Freenode for reporting this issue to me.


Discussion:

Resolving patch:
diff --git a/bodhi/server/scripts/dequeue_stable.py b/bodhi/server/scripts/dequeue_stable.py
--- a/bodhi/server/scripts/dequeue_stable.py
+++ b/bodhi/server/scripts/dequeue_stable.py
@@ -1,5 +1,5 @@
 # -*- coding: utf-8 -*-
-# Copyright © 2017 Caleigh Runge-Hottman
+# Copyright © 2017 Caleigh Runge-Hottman and Red Hat, Inc.
 #
 # This file is part of Bodhi.
 #
@@ -36,11 +36,17 @@ def dequeue_stable():
     try:
         batched = db.query(models.Update).filter_by(request=models.UpdateRequest.batched).all()
         for update in batched:
-            update.set_request(db, models.UpdateRequest.stable, u'bodhi')
-        db.commit()
-
+            try:
+                update.set_request(db, models.UpdateRequest.stable, u'bodhi')
+                db.commit()
+            except Exception as e:
+                print('Unable to stabilize {}: {}'.format(update.alias, str(e)))
+                db.rollback()
+                msg = u"Bodhi is unable to request this update for stabilization: {}"
+                update.comment(db, msg.format(str(e)), author=u'bodhi')
+                db.commit()
     except Exception as e:
         print(str(e))
-        db.rollback()
-        Session.remove()
         sys.exit(1)
+    finally:
+        Session.remove()


Test patch:
diff --git a/bodhi/tests/server/scripts/test_dequeue_stable.py b/bodhi/tests/server/scripts/test_dequeue_stable.py
--- a/bodhi/tests/server/scripts/test_dequeue_stable.py
+++ b/bodhi/tests/server/scripts/test_dequeue_stable.py
@@ -1,5 +1,5 @@
 # -*- coding: utf-8 -*-
-# Copyright © 2017 Caleigh Runge-Hottman
+# Copyright © 2017 Caleigh Runge-Hottman and Red Hat, Inc.
 #
 # This file is part of Bodhi.
 #
@@ -22,17 +22,57 @@
 from datetime import datetime, timedelta
 
 from click import testing
+import mock
 
-from bodhi.server import models
+from bodhi.server import config, models
 from bodhi.server.scripts import dequeue_stable
 from bodhi.tests.server.base import BaseTestCase
 
 
+@mock.patch('bodhi.server.scripts.dequeue_stable.Session.remove')
 class TestDequeueStable(BaseTestCase):
     """
     This class contains tests for the dequeue_stable() function.
     """
-    def test_dequeue_stable(self):
+    def test_bad_update_doesnt_block_others(self, remove):
+        """
+        Assert that one bad update in the set of batched updates doesn't block the others.
+        """
+        runner = testing.CliRunner()
+        update_1 = models.Update.query.first()
+        update_1_title = update_1.title
+        update_1.request = models.UpdateRequest.batched
+        update_1.locked = False
+        update_1.date_testing = datetime.utcnow() - timedelta(days=7)
+        build_2 = models.RpmBuild(nvr='bodhi-3.1.0-1.fc17', package=update_1.builds[0].package)
+        self.db.add(build_2)
+        update_2 = models.Update(
+            title=build_2.nvr, builds=[build_2], type=models.UpdateType.enhancement, notes='blah',
+            status=models.UpdateStatus.testing, request=models.UpdateRequest.batched,
+            user=update_1.user, release=update_1.release)
+        update_2_title = update_2.title
+        self.db.add(update_2)
+        self.db.commit()
+
+        result = runner.invoke(dequeue_stable.dequeue_stable, [])
+
+        self.assertEqual(result.exit_code, 0)
+        self.assertTrue(config.config['not_yet_tested_msg'] in result.output)
+        self.assertEqual(models.Update.query.filter_by(title=update_1_title).one().request,
+                         models.UpdateRequest.stable)
+        # set_request() rejected this update since it hasn't met testing requirements.
+        self.assertEqual(models.Update.query.filter_by(title=update_2_title).one().request,
+                         models.UpdateRequest.batched)
+        # It should also have received a stern comment from Bodhi.
+        c = models.Update.query.filter_by(title=update_2_title).one().comments[-1]
+        self.assertEqual(c.user.name, 'bodhi')
+        self.assertEqual(
+            c.text,
+            'Bodhi is unable to request this update for stabilization: {}'.format(
+                config.config['not_yet_tested_msg']))
+        remove.assert_called_once_with()
+
+    def test_dequeue_stable(self, remove):
         """
         Assert that dequeue_stable moves only the batched updates to stable.
         """
@@ -49,17 +89,22 @@ def test_dequeue_stable(self):
 
         update = self.db.query(models.Update).all()[0]
         self.assertEqual(update.request, models.UpdateRequest.stable)
+        remove.assert_called_once_with()
 
-    def test_dequeue_stable_exception(self):
+    def test_dequeue_stable_exception(self, remove):
         """
-        Assert that a locked update triggers an exception, and doesn't move to stable.
+        Assert that a failure to query the db is printed and raised.
         """
         runner = testing.CliRunner()
         update = self.db.query(models.Update).all()[0]
         update.request = models.UpdateRequest.batched
         self.db.commit()
 
-        result = runner.invoke(dequeue_stable.dequeue_stable, [])
+        # Let's simulate an error connecting to the db.
+        with mock.patch('bodhi.server.scripts.dequeue_stable.Session') as Session:
+            Session.return_value.query.side_effect = IOError("You can't haz network.")
+            result = runner.invoke(dequeue_stable.dequeue_stable, [])
 
         self.assertEqual(result.exit_code, 1)
-        self.assertEqual(result.output, u"Can't change the request on a locked update\n")
+        self.assertEqual(result.output, "You can't haz network.\n")
+        Session.remove.assert_called_once_with()