Repository: fedora-infra/bodhi

Issue:
Race condition between robosignatory signing builds and Builds being added to the db
The signed handler often sees messages from robosignatory that it has signed a build, but then the Build isn't found in Bodhi's database. This is due to the Build not being committed before Bodhi tells Koji to add the build to the pre-signing tag. This causes updates to never make it to the signed state, which causes the masher to never move them to the updates-testing repository.

The fix itself will be easy, but we will need to expend a lot of effort in testing the fix since it is unlikely that there are tests for the specific function in question.
Race condition between robosignatory signing builds and Builds being added to the db
The signed handler often sees messages from robosignatory that it has signed a build, but then the Build isn't found in Bodhi's database. This is due to the Build not being committed before Bodhi tells Koji to add the build to the pre-signing tag. This causes updates to never make it to the signed state, which causes the masher to never move them to the updates-testing repository.

The fix itself will be easy, but we will need to expend a lot of effort in testing the fix since it is unlikely that there are tests for the specific function in question.


Discussion:

Resolving patch:
diff --git a/bodhi/server/services/updates.py b/bodhi/server/services/updates.py
--- a/bodhi/server/services/updates.py
+++ b/bodhi/server/services/updates.py
@@ -18,6 +18,7 @@
 from cornice import Service
 from sqlalchemy import func, distinct
 from sqlalchemy.sql import or_
+import transaction
 
 from bodhi.server import log
 from bodhi.server.exceptions import BodhiException, LockedUpdateException
@@ -351,11 +352,8 @@ def new_update(request):
         # Create the Package and Build entities
         for nvr in data['builds']:
             name, version, release = request.buildinfo[nvr]['nvr']
-            package = request.db.query(Package).filter_by(name=name).first()
-            if not package:
-                package = Package(name=name)
-                request.db.add(package)
-                request.db.flush()
+            # The package will have been created by validate_acls
+            package = request.db.query(Package).filter_by(name=name).one()
 
             build = Build.get(nvr, request.db)
 
@@ -370,6 +368,13 @@ def new_update(request):
             builds.append(build)
             releases.add(request.buildinfo[build.nvr]['release'])
 
+        # We want to go ahead and commit the transaction now so that the Builds are in the database.
+        # Otherwise, there will be a race condition between robosignatory signing the Builds and the
+        # signed handler attempting to mark the builds as signed. When we lose that race, the signed
+        # handler doesn't see the Builds in the database and gives up. After that, nothing will mark
+        # the builds as signed.
+        transaction.commit()
+
         if data.get('edited'):
 
             log.info('Editing update: %s' % data['edited'])


Test patch:
diff --git a/bodhi/tests/server/base.py b/bodhi/tests/server/base.py
--- a/bodhi/tests/server/base.py
+++ b/bodhi/tests/server/base.py
@@ -22,6 +22,7 @@
 from sqlalchemy import event
 from sqlalchemy.orm import scoped_session, sessionmaker
 from zope.sqlalchemy import ZopeTransactionExtension
+import transaction
 
 from bodhi.server import log
 from bodhi.tests.server import populate
@@ -37,12 +38,13 @@ class BaseTestCase(unittest.TestCase):
     def setUp(self):
         engine = create_engine(DB_PATH)
         # We want a special session that lasts longer than a transaction
-        Session = scoped_session(
+        self.Session = scoped_session(
             sessionmaker(bind=engine, extension=ZopeTransactionExtension(keep_session=True)))
         log.debug('Creating all models for %s' % engine)
         Base.metadata.bind = engine
         Base.metadata.create_all(engine)
-        self.db = Session()
+        self.db = self.Session()
+        transaction.begin()
         populate(self.db)
 
         # Track sql statements in every test
@@ -54,6 +56,8 @@ def track(conn, cursor, statement, param, ctx, many):
         event.listen(engine, "before_cursor_execute", track)
 
     def tearDown(self):
+        transaction.abort()
         log.debug('Removing session')
         self.db.close()
+        self.Session.remove()
         del self.sql_statements
diff --git a/bodhi/tests/server/services/test_updates.py b/bodhi/tests/server/services/test_updates.py
--- a/bodhi/tests/server/services/test_updates.py
+++ b/bodhi/tests/server/services/test_updates.py
@@ -303,6 +303,31 @@ def test_new_update_with_existing_build(self, *args):
 
         eq_(resp.json['title'], 'bodhi-2.0.0-3.fc17')
 
+    @mock.patch(**mock_valid_requirements)
+    def test_new_update_with_existing_package(self, *args):
+        """Test submitting a new update with a package that is already in the database."""
+        package = Package(name='existing-package')
+        self.db.add(package)
+        self.db.flush()
+        args = self.get_update(u'existing-package-2.4.1-5.fc17')
+
+        resp = self.app.post_json('/updates/', args)
+
+        eq_(resp.json['title'], 'existing-package-2.4.1-5.fc17')
+        package = self.db.query(Package).filter_by(name=u'existing-package').one()
+        self.assertEqual(package.name, 'existing-package')
+
+    @mock.patch(**mock_valid_requirements)
+    def test_new_update_with_missing_package(self, *args):
+        """Test submitting a new update with a package that is not already in the database."""
+        args = self.get_update(u'missing-package-2.4.1-5.fc17')
+
+        resp = self.app.post_json('/updates/', args)
+
+        eq_(resp.json['title'], 'missing-package-2.4.1-5.fc17')
+        package = self.db.query(Package).filter_by(name=u'missing-package').one()
+        self.assertEqual(package.name, 'missing-package')
+
     @mock.patch(**mock_valid_requirements)
     @mock.patch('bodhi.server.notifications.publish')
     def test_cascade_package_requirements_to_update(self, publish, *args):
@@ -379,6 +404,58 @@ def test_obsoletion(self, publish, *args):
             urlparse.urljoin(config['base_address'], '/updates/FEDORA-2016-53345602d5'))
         self.assertEquals(up.comments[-1].text, expected_comment)
 
+    @mock.patch(**mock_valid_requirements)
+    @mock.patch('bodhi.server.notifications.publish')
+    @mock.patch('bodhi.server.services.updates.Update.new', side_effect=IOError('oops!'))
+    def test_unexpected_exception(self, publish, *args):
+        """Ensure that an unexpected Exception is handled by new_update()."""
+        update = self.get_update('bodhi-2.3.2-1.fc17')
+
+        r = self.app.post_json('/updates/', update, status=400)
+
+        self.assertEquals(r.json_body['status'], 'error')
+        self.assertEquals(r.json_body['errors'][0]['description'],
+                          "Unable to create update.  oops!")
+        # Despite the Exception, the Build should still exist in the database
+        build = self.db.query(Build).filter(Build.nvr == u'bodhi-2.3.2-1.fc17').one()
+        self.assertEqual(build.package.name, 'bodhi')
+
+    @mock.patch(**mock_valid_requirements)
+    @mock.patch('bodhi.server.services.updates.Update.obsolete_older_updates',
+                side_effect=RuntimeError("bet you didn't see this coming!"))
+    def test_obsoletion_with_exception(self, *args):
+        """
+        Assert that an exception during obsoletion is properly handled.
+        """
+        nvr = 'bodhi-2.0.0-2.fc17'
+        args = self.get_update(nvr)
+        with mock.patch(**mock_uuid4_version1):
+            self.app.post_json('/updates/', args)
+        up = self.db.query(Update).filter_by(title=nvr).one()
+        up.status = UpdateStatus.testing
+        up.request = None
+        args = self.get_update('bodhi-2.0.0-3.fc17')
+
+        with mock.patch(**mock_uuid4_version2):
+            r = self.app.post_json('/updates/', args).json_body
+
+        self.assertEquals(r['request'], 'testing')
+        # The exception handler should have put an error message in the caveats.
+        self.assertEquals(r['caveats'][0]['description'],
+                          "Problem obsoleting older updates: bet you didn't see this coming!")
+        # Check for the comment multiple ways. The comment will be about the update being submitted
+        # for testing instead of being about the obsoletion, since the obsoletion failed.
+        # Note that caveats above don't support markdown, but comments do.
+        expected_comment = 'This update has been submitted for testing by guest. '
+        expected_comment = expected_comment.format(
+            urlparse.urljoin(config['base_address'], '/updates/FEDORA-2016-033713b73b'))
+        self.assertEquals(r['comments'][-1]['text'], expected_comment)
+        up = self.db.query(Update).filter_by(title=nvr).one()
+        # The old update failed to get obsoleted.
+        self.assertEquals(up.status, UpdateStatus.testing)
+        expected_comment = u'This update has been submitted for testing by guest. '
+        self.assertEquals(up.comments[-1].text, expected_comment)
+
 
 class TestUpdatesService(bodhi.tests.server.functional.base.BaseWSGICase):