Repository: fedora-infra/bodhi

Issue:
Bodhi doesn't always close its database connections after requests
I noticed last night in staging that Bodhi was sometimes logging this traceback:

```
[Wed May 23 13:39:04.795112 2018] [wsgi:error] [pid 322:tid 140371350968064] [client 10.128.2.1:41472] Traceback (most recent call last):
[Wed May 23 13:39:04.795141 2018] [wsgi:error] [pid 322:tid 140371350968064] [client 10.128.2.1:41472]   File "/usr/lib/python2.7/site-packages/pyramid/tweens.py", line 39, in excview_tween
[Wed May 23 13:39:04.795164 2018] [wsgi:error] [pid 322:tid 140371350968064] [client 10.128.2.1:41472]     response = handler(request)
[Wed May 23 13:39:04.795185 2018] [wsgi:error] [pid 322:tid 140371350968064] [client 10.128.2.1:41472]   File "/usr/lib/python2.7/site-packages/pyramid/router.py", line 156, in handle_request
[Wed May 23 13:39:04.795207 2018] [wsgi:error] [pid 322:tid 140371350968064] [client 10.128.2.1:41472]     view_name
[Wed May 23 13:39:04.795229 2018] [wsgi:error] [pid 322:tid 140371350968064] [client 10.128.2.1:41472]   File "/usr/lib/python2.7/site-packages/pyramid/view.py", line 617, in _call_view
[Wed May 23 13:39:04.795251 2018] [wsgi:error] [pid 322:tid 140371350968064] [client 10.128.2.1:41472]     response = view_callable(context, request)
[Wed May 23 13:39:04.795272 2018] [wsgi:error] [pid 322:tid 140371350968064] [client 10.128.2.1:41472]   File "/usr/lib/python2.7/site-packages/pyramid/config/views.py", line 181, in __call__
[Wed May 23 13:39:04.795294 2018] [wsgi:error] [pid 322:tid 140371350968064] [client 10.128.2.1:41472]     return view(context, request)
[Wed May 23 13:39:04.795315 2018] [wsgi:error] [pid 322:tid 140371350968064] [client 10.128.2.1:41472]   File "/usr/lib/python2.7/site-packages/pyramid/viewderivers.py", line 389, in attr_view
[Wed May 23 13:39:04.795337 2018] [wsgi:error] [pid 322:tid 140371350968064] [client 10.128.2.1:41472]     return view(context, request)
[Wed May 23 13:39:04.795368 2018] [wsgi:error] [pid 322:tid 140371350968064] [client 10.128.2.1:41472]   File "/usr/lib/python2.7/site-packages/pyramid/viewderivers.py", line 367, in predicate_wrapper
[Wed May 23 13:39:04.795389 2018] [wsgi:error] [pid 322:tid 140371350968064] [client 10.128.2.1:41472]     return view(context, request)
[Wed May 23 13:39:04.795409 2018] [wsgi:error] [pid 322:tid 140371350968064] [client 10.128.2.1:41472]   File "/usr/lib/python2.7/site-packages/pyramid/viewderivers.py", line 300, in secured_view
[Wed May 23 13:39:04.795429 2018] [wsgi:error] [pid 322:tid 140371350968064] [client 10.128.2.1:41472]     return view(context, request)
[Wed May 23 13:39:04.795449 2018] [wsgi:error] [pid 322:tid 140371350968064] [client 10.128.2.1:41472]   File "/usr/lib/python2.7/site-packages/pyramid/viewderivers.py", line 438, in rendered_view
[Wed May 23 13:39:04.795470 2018] [wsgi:error] [pid 322:tid 140371350968064] [client 10.128.2.1:41472]     result = view(context, request)
[Wed May 23 13:39:04.795490 2018] [wsgi:error] [pid 322:tid 140371350968064] [client 10.128.2.1:41472]   File "/usr/lib/python2.7/site-packages/pyramid/viewderivers.py", line 147, in _requestonly_view
[Wed May 23 13:39:04.795510 2018] [wsgi:error] [pid 322:tid 140371350968064] [client 10.128.2.1:41472]     response = view(request)
[Wed May 23 13:39:04.795530 2018] [wsgi:error] [pid 322:tid 140371350968064] [client 10.128.2.1:41472]   File "/usr/lib/python2.7/site-packages/cornice/service.py", line 494, in wrapper
[Wed May 23 13:39:04.795551 2018] [wsgi:error] [pid 322:tid 140371350968064] [client 10.128.2.1:41472]     response = view_()
[Wed May 23 13:39:04.795571 2018] [wsgi:error] [pid 322:tid 140371350968064] [client 10.128.2.1:41472]   File "/usr/lib/python2.7/site-packages/bodhi/server/services/composes.py", line 74, in collection_get
[Wed May 23 13:39:04.795601 2018] [wsgi:error] [pid 322:tid 140371350968064] [client 10.128.2.1:41472]     return {'composes': sorted(models.Compose.query.all())}
[Wed May 23 13:39:04.795622 2018] [wsgi:error] [pid 322:tid 140371350968064] [client 10.128.2.1:41472]   File "/usr/lib64/python2.7/site-packages/sqlalchemy/orm/query.py", line 2703, in all
[Wed May 23 13:39:04.795644 2018] [wsgi:error] [pid 322:tid 140371350968064] [client 10.128.2.1:41472]     return list(self)
[Wed May 23 13:39:04.795664 2018] [wsgi:error] [pid 322:tid 140371350968064] [client 10.128.2.1:41472]   File "/usr/lib64/python2.7/site-packages/sqlalchemy/orm/query.py", line 2855, in __iter__
[Wed May 23 13:39:04.795684 2018] [wsgi:error] [pid 322:tid 140371350968064] [client 10.128.2.1:41472]     return self._execute_and_instances(context)
[Wed May 23 13:39:04.795704 2018] [wsgi:error] [pid 322:tid 140371350968064] [client 10.128.2.1:41472]   File "/usr/lib64/python2.7/site-packages/sqlalchemy/orm/query.py", line 2876, in _execute_and_instances
[Wed May 23 13:39:04.795725 2018] [wsgi:error] [pid 322:tid 140371350968064] [client 10.128.2.1:41472]     close_with_result=True)
[Wed May 23 13:39:04.795745 2018] [wsgi:error] [pid 322:tid 140371350968064] [client 10.128.2.1:41472]   File "/usr/lib64/python2.7/site-packages/sqlalchemy/orm/query.py", line 2885, in _get_bind_args
[Wed May 23 13:39:04.795765 2018] [wsgi:error] [pid 322:tid 140371350968064] [client 10.128.2.1:41472]     **kw
[Wed May 23 13:39:04.795785 2018] [wsgi:error] [pid 322:tid 140371350968064] [client 10.128.2.1:41472]   File "/usr/lib64/python2.7/site-packages/sqlalchemy/orm/query.py", line 2867, in _connection_from_session
[Wed May 23 13:39:04.795806 2018] [wsgi:error] [pid 322:tid 140371350968064] [client 10.128.2.1:41472]     conn = self.session.connection(**kw)
[Wed May 23 13:39:04.795826 2018] [wsgi:error] [pid 322:tid 140371350968064] [client 10.128.2.1:41472]   File "/usr/lib64/python2.7/site-packages/sqlalchemy/orm/session.py", line 1019, in connection
[Wed May 23 13:39:04.795846 2018] [wsgi:error] [pid 322:tid 140371350968064] [client 10.128.2.1:41472]     execution_options=execution_options)
[Wed May 23 13:39:04.795866 2018] [wsgi:error] [pid 322:tid 140371350968064] [client 10.128.2.1:41472]   File "/usr/lib64/python2.7/site-packages/sqlalchemy/orm/session.py", line 1024, in _connection_for_bind
[Wed May 23 13:39:04.795886 2018] [wsgi:error] [pid 322:tid 140371350968064] [client 10.128.2.1:41472]     engine, execution_options)
[Wed May 23 13:39:04.795906 2018] [wsgi:error] [pid 322:tid 140371350968064] [client 10.128.2.1:41472]   File "/usr/lib64/python2.7/site-packages/sqlalchemy/orm/session.py", line 409, in _connection_for_bind
[Wed May 23 13:39:04.795927 2018] [wsgi:error] [pid 322:tid 140371350968064] [client 10.128.2.1:41472]     conn = bind.contextual_connect()
[Wed May 23 13:39:04.795947 2018] [wsgi:error] [pid 322:tid 140371350968064] [client 10.128.2.1:41472]   File "/usr/lib64/python2.7/site-packages/sqlalchemy/engine/base.py", line 2112, in contextual_connect
[Wed May 23 13:39:04.795967 2018] [wsgi:error] [pid 322:tid 140371350968064] [client 10.128.2.1:41472]     self._wrap_pool_connect(self.pool.connect, None),
[Wed May 23 13:39:04.796020 2018] [wsgi:error] [pid 322:tid 140371350968064] [client 10.128.2.1:41472]   File "/usr/lib64/python2.7/site-packages/sqlalchemy/engine/base.py", line 2147, in _wrap_pool_connect
[Wed May 23 13:39:04.796065 2018] [wsgi:error] [pid 322:tid 140371350968064] [client 10.128.2.1:41472]     return fn()
[Wed May 23 13:39:04.796090 2018] [wsgi:error] [pid 322:tid 140371350968064] [client 10.128.2.1:41472]   File "/usr/lib64/python2.7/site-packages/sqlalchemy/pool.py", line 387, in connect
[Wed May 23 13:39:04.796112 2018] [wsgi:error] [pid 322:tid 140371350968064] [client 10.128.2.1:41472]     return _ConnectionFairy._checkout(self)
[Wed May 23 13:39:04.796140 2018] [wsgi:error] [pid 322:tid 140371350968064] [client 10.128.2.1:41472]   File "/usr/lib64/python2.7/site-packages/sqlalchemy/pool.py", line 768, in _checkout
[Wed May 23 13:39:04.796167 2018] [wsgi:error] [pid 322:tid 140371350968064] [client 10.128.2.1:41472]     fairy = _ConnectionRecord.checkout(pool)
[Wed May 23 13:39:04.796193 2018] [wsgi:error] [pid 322:tid 140371350968064] [client 10.128.2.1:41472]   File "/usr/lib64/python2.7/site-packages/sqlalchemy/pool.py", line 516, in checkout
[Wed May 23 13:39:04.796219 2018] [wsgi:error] [pid 322:tid 140371350968064] [client 10.128.2.1:41472]     rec = pool._do_get()
[Wed May 23 13:39:04.796244 2018] [wsgi:error] [pid 322:tid 140371350968064] [client 10.128.2.1:41472]   File "/usr/lib64/python2.7/site-packages/sqlalchemy/pool.py", line 1133, in _do_get
[Wed May 23 13:39:04.796270 2018] [wsgi:error] [pid 322:tid 140371350968064] [client 10.128.2.1:41472]     (self.size(), self.overflow(), self._timeout))
[Wed May 23 13:39:04.796295 2018] [wsgi:error] [pid 322:tid 140371350968064] [client 10.128.2.1:41472] TimeoutError: QueuePool limit of size 5 overflow 10 reached, connection timed out, timeout 30
```

It seemed odd at first, since the particular rev of the beta only altered code in the composer and not in the web requests.

Thanks to a co-debugging session with @puiterwijk, we came to learn that Bodhi does not call (```get_db_session_for_request()```](https://github.com/fedora-infra/bodhi/blob/3.7.0/bodhi/server/__init__.py#L50-L83) unless something during that request accessed ```request.db``` at least once. The 3.8.0 release has a patch that refactored the base model's ```get()``` method so it stopped needing a db session passed to it, and this caused the ```/composes/``` view to no longer perform any action that accesses ```request.db```. It turns out that this problem has been in all 3.8.0 betas, but I didn't notice it until last night because yesterday was the first time I set a watch on ```bodhi composes list```, which calls that API and drains connections from the pool without returning them.

The fix is simple and cleaner: we just need to [subscribe to Pyramid's ```NewRequest``` event](https://docs.pylonsproject.org/projects/pyramid/en/latest/narr/webob.html#cleaning-up-after-a-request) so we can add the call to ```cleanup()```, rather than relying on ```request.db``` to get used during the request.

This is a 3.8.0 blocker, so it is critical priority.


Discussion:

Resolving patch:
diff --git a/bodhi/server/__init__.py b/bodhi/server/__init__.py
--- a/bodhi/server/__init__.py
+++ b/bodhi/server/__init__.py
@@ -1,5 +1,5 @@
 # -*- coding: utf-8 -*-
-# Copyright © 2007-2017 Red Hat, Inc. and others.
+# Copyright © 2007-2018 Red Hat, Inc. and others.
 #
 # This file is part of Bodhi.
 #
@@ -47,42 +47,6 @@
 # Request methods
 #
 
-def get_db_session_for_request(request=None):
-    """
-    Return a database session that is meant to be used for the given request.
-
-    It handles rolling back or committing the session based on whether an exception occurred or
-    not. To get a database session that's not tied to the request/response cycle, just use the
-    :data:`Session` scoped session in this module.
-
-    Args:
-        request (pyramid.request.Request): The request object to create a session for.
-
-    Returns:
-        sqlalchemy.orm.session.Session: A database session.
-    """
-    session = request.registry.sessionmaker()
-
-    def cleanup(request):
-        """
-        Commit the database changes if no exceptions occurred.
-
-        This is a post-request hook.
-
-        Args:
-            request (pyramid.request.Request): The current web request.
-        """
-        if request.exception is not None:
-            session.rollback()
-        else:
-            session.commit()
-        session.close()
-
-    request.add_finished_callback(cleanup)
-
-    return session
-
-
 def get_cacheregion(request):
     """
     Return a CacheRegion to be used to cache results.
@@ -245,7 +209,8 @@ def main(global_config, testing=None, session=None, **settings):
     Args:
         global_config (dict): A dictionary with two keys: __file__, a path to the ini file, and
             here, the path to the code.
-        testing (bool or None): Whether or not we are in testing mode.
+        testing (str or None): If this app is contructed by the unit tests, they should set this to
+            a username.
         session (sqlalchemy.orm.session.Session or None): If given, the session will be used instead
             of building a new one.
         settings (dictionary): Unused.
@@ -286,7 +251,7 @@ def main(global_config, testing=None, session=None, **settings):
     else:
         config.registry.sessionmaker = Session
 
-    config.add_request_method(get_db_session_for_request, 'db', reify=True)
+    config.add_request_method(lambda x: Session, 'db', reify=True)
 
     config.add_request_method(get_user, 'user', reify=True)
     config.add_request_method(get_koji, 'koji', reify=True)
@@ -354,6 +319,7 @@ def main(global_config, testing=None, session=None, **settings):
     config.scan('bodhi.server.views')
     config.scan('bodhi.server.services')
     config.scan('bodhi.server.captcha')
+    config.scan('bodhi.server.webapp')
 
     # Though importing in the middle of this function is the darkest of evils, we cannot do it any
     # other way without a backwards-incompatible change. See
@@ -376,5 +342,8 @@ def main(global_config, testing=None, session=None, **settings):
     # return value.
     generic._generate_home_page_stats()
 
+    # Let's close out the db session we used to warm the caches.
+    Session.remove()
+
     log.info('Bodhi ready and at your service!')
     return config.make_wsgi_app()
diff --git a/bodhi/server/webapp.py b/bodhi/server/webapp.py
new file mode 100644
--- /dev/null
+++ b/bodhi/server/webapp.py
@@ -0,0 +1,73 @@
+# -*- coding: utf-8 -*-
+# Copyright © 2018 Red Hat, Inc.
+#
+# This file is part of Bodhi.
+#
+# This program is free software; you can redistribute it and/or
+# modify it under the terms of the GNU General Public License
+# as published by the Free Software Foundation; either version 2
+# of the License, or (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
+"""
+Define Bodhi's WSGI application.
+
+As of the writing of this docblock, this module is a bit misnamed since the webapp is actually
+defined in bodhi.server.__init__. However, that is an anti-pattern with lots of nasty in-line
+imports due to circular dependencies, and this module is intended to solve that problem.
+Unfortunately, it is a backwards-incompatible change to move main() here, so it will remain in
+__init__ until we make a major Bodhi release. See https://github.com/fedora-infra/bodhi/issues/2294
+"""
+
+from pyramid.events import NewRequest, subscriber
+
+from bodhi import server
+
+
+def _complete_database_session(request):
+    """
+    Commit the database changes if no exceptions occurred.
+
+    This is a post-request hook. It handles rolling back or committing the session based on whether
+    an exception occurred or not. To get a database session that's not tied to the request/response
+    cycle, just use the :data:`Session` scoped session.
+
+    Args:
+        request (pyramid.request.Request): The current web request.
+    """
+    _rollback_or_commit(request)
+    server.Session().close()
+    server.Session.remove()
+
+
+@subscriber(NewRequest)
+def _prepare_request(event):
+    """
+    Add callbacks onto every new request.
+
+    This function adds a callback to clean up the database session when the request is finished.
+
+    Args:
+        event (pyramid.events.NewRequest): The new request event.
+    """
+    event.request.add_finished_callback(_complete_database_session)
+
+
+def _rollback_or_commit(request):
+    """
+    Commit the transaction if there are no exceptions, otherwise rollback.
+
+    Args:
+        request (pyramid.request.Request): The current web request.
+    """
+    if request.exception is not None:
+        server.Session().rollback()
+    else:
+        server.Session().commit()


Test patch:
diff --git a/bodhi/tests/server/base.py b/bodhi/tests/server/base.py
--- a/bodhi/tests/server/base.py
+++ b/bodhi/tests/server/base.py
@@ -25,7 +25,7 @@
 from sqlalchemy import event
 import mock
 
-from bodhi.server import bugs, buildsys, models, initialize_db, Session, config, main
+from bodhi.server import bugs, buildsys, models, initialize_db, Session, config, main, webapp
 from bodhi.tests.server import create_update, populate
 
 
@@ -134,6 +134,7 @@ class BaseTestCase(unittest.TestCase):
         'cors_connect_src': 'http://0.0.0.0:6543',
         'cors_origins_ro': 'http://0.0.0.0:6543',
         'cors_origins_rw': 'http://0.0.0.0:6543',
+        'sqlalchemy.url': DEFAULT_DB
     }
 
     def setUp(self):
@@ -163,21 +164,8 @@ def setUp(self):
         bugs.set_bugtracker()
         buildsys.setup_buildsystem({'buildsystem': 'dev'})
 
-        def request_db(request=None):
-            """
-            Replace the db session function with one that doesn't close the session.
-
-            This allows tests to make assertions about the database. Without it, all
-            the changes would be rolled back to when the nested transaction is started.
-            """
-            def cleanup(request):
-                if request.exception is not None:
-                    Session().rollback()
-                else:
-                    Session().commit()
-            request.add_finished_callback(cleanup)
-            return Session()
-        self._request_sesh = mock.patch('bodhi.server.get_db_session_for_request', request_db)
+        self._request_sesh = mock.patch('bodhi.server.webapp._complete_database_session',
+                                        webapp._rollback_or_commit)
         self._request_sesh.start()
 
         # Create the test WSGI app one time. We should avoid creating too many
@@ -186,7 +174,11 @@ def cleanup(request):
         # out how to make Pyramid forget about these.
         global _app
         if _app is None:
-            _app = TestApp(main({}, testing=u'guest', **self.app_settings))
+            # We don't want to call Session.remove() during the unit tests, because that will
+            # trigger the restart_savepoint() callback defined above which will remove the data
+            # added by populate().
+            with mock.patch('bodhi.server.Session.remove'):
+                _app = TestApp(main({}, testing=u'guest', **self.app_settings))
         self.app = _app
 
     def get_csrf_token(self, app=None):
diff --git a/bodhi/tests/server/services/test_comments.py b/bodhi/tests/server/services/test_comments.py
--- a/bodhi/tests/server/services/test_comments.py
+++ b/bodhi/tests/server/services/test_comments.py
@@ -258,7 +258,8 @@ def test_anonymous_commenting_with_no_author(self):
             'authtkt.secret': 'whatever',
             'authtkt.secure': True,
         })
-        app = TestApp(main({}, session=self.db, **anonymous_settings))
+        with mock.patch('bodhi.server.Session.remove'):
+            app = TestApp(main({}, session=self.db, **anonymous_settings))
 
         comment = {
             u'update': 'bodhi-2.0-1.fc17',
diff --git a/bodhi/tests/server/services/test_overrides.py b/bodhi/tests/server/services/test_overrides.py
--- a/bodhi/tests/server/services/test_overrides.py
+++ b/bodhi/tests/server/services/test_overrides.py
@@ -661,7 +661,9 @@ def test_override_view_not_loggedin(self):
             'authtkt.secret': 'whatever',
             'authtkt.secure': True,
         })
-        app = TestApp(main({}, session=self.db, **anonymous_settings))
+        with mock.patch('bodhi.server.Session.remove'):
+            app = TestApp(main({}, session=self.db, **anonymous_settings))
+
         resp = app.get('/overrides/bodhi-2.0-1.fc17',
                        status=200, headers={'Accept': 'text/html'})
         self.assertNotIn('<span>New Buildroot Override Form Requires JavaScript</span>', resp)
diff --git a/bodhi/tests/server/services/test_releases.py b/bodhi/tests/server/services/test_releases.py
--- a/bodhi/tests/server/services/test_releases.py
+++ b/bodhi/tests/server/services/test_releases.py
@@ -55,7 +55,9 @@ def test_anonymous_cant_edit_release(self):
         """Ensure that an unauthenticated user cannot edit a release, since only an admin should."""
         name = u"F22"
         # Create a new app so we are the anonymous user.
-        app = webtest.TestApp(server.main({}, session=self.db, **self.app_settings))
+        with mock.patch('bodhi.server.Session.remove'):
+            app = webtest.TestApp(server.main({}, session=self.db, **self.app_settings))
+
         res = app.get('/releases/%s' % name, status=200)
         r = res.json_body
         r["edited"] = name
diff --git a/bodhi/tests/server/services/test_updates.py b/bodhi/tests/server/services/test_updates.py
--- a/bodhi/tests/server/services/test_updates.py
+++ b/bodhi/tests/server/services/test_updates.py
@@ -171,7 +171,8 @@ def test_no_privs(self, publish, *args):
         user = User(name=u'bodhi')
         self.db.add(user)
         self.db.commit()
-        app = TestApp(main({}, testing=u'bodhi', session=self.db, **self.app_settings))
+        with mock.patch('bodhi.server.Session.remove'):
+            app = TestApp(main({}, testing=u'bodhi', session=self.db, **self.app_settings))
         update_json = self.get_update(u'bodhi-2.1-1.fc17')
         update_json['csrf_token'] = self.get_csrf_token(app)
 
@@ -199,7 +200,8 @@ def test_provenpackager_privs(self, publish, *args):
         group = self.db.query(Group).filter_by(name=u'provenpackager').one()
         user.groups.append(group)
 
-        app = TestApp(main({}, testing=u'bodhi', session=self.db, **self.app_settings))
+        with mock.patch('bodhi.server.Session.remove'):
+            app = TestApp(main({}, testing=u'bodhi', session=self.db, **self.app_settings))
         update = self.get_update(u'bodhi-2.1-1.fc17')
         update['csrf_token'] = app.get('/csrf').json_body['csrf_token']
         res = app.post_json('/updates/', update)
@@ -213,13 +215,13 @@ def test_provenpackager_privs(self, publish, *args):
     @mock.patch(**mock_valid_requirements)
     def test_pkgdb_outage(self, *args):
         "Test the case where our call to the pkgdb throws an exception"
-        settings = self.app_settings.copy()
-        settings['acl_system'] = 'pkgdb'
-        settings['pkgdb_url'] = 'invalidurl'
-        app = TestApp(main({}, testing=u'guest', session=self.db, **settings))
         update = self.get_update(u'bodhi-2.0-2.fc17')
-        update['csrf_token'] = app.get('/csrf').json_body['csrf_token']
-        res = app.post_json('/updates/', update, status=400)
+        update['csrf_token'] = self.get_csrf_token()
+
+        with mock.patch.dict('bodhi.server.validators.config',
+                             {'acl_system': 'pkgdb', 'pkgdb_url': 'invalidurl'}):
+            res = self.app.post_json('/updates/', update, status=400)
+
         assert "Unable to access the Package Database" in res, res
 
     @unittest.skipIf(six.PY3, 'Not working with Python 3 yet')
@@ -773,7 +775,9 @@ def test_edit_without_permission(self):
         """
         Test a logged in User without permissions on the update can't see the form
         """
-        app = TestApp(main({}, testing=u'anonymous', session=self.db, **self.app_settings))
+        with mock.patch('bodhi.server.Session.remove'):
+            app = TestApp(main({}, testing=u'anonymous', session=self.db, **self.app_settings))
+
         resp = app.get(
             '/updates/FEDORA-{}-a3bbe1a8f2/edit'.format(datetime.utcnow().year), status=400)
         self.assertIn(
@@ -967,7 +971,8 @@ def test_provenpackager_edit_anything(self, publish, *args):
         group2 = self.db.query(Group).filter_by(name=u'packager').one()
         user2.groups.append(group2)
 
-        app = TestApp(main({}, testing=u'ralph', session=self.db, **self.app_settings))
+        with mock.patch('bodhi.server.Session.remove'):
+            app = TestApp(main({}, testing=u'ralph', session=self.db, **self.app_settings))
         up_data = self.get_update(nvr)
         up_data['csrf_token'] = app.get('/csrf').json_body['csrf_token']
         res = app.post_json('/updates/', up_data)
@@ -975,7 +980,8 @@ def test_provenpackager_edit_anything(self, publish, *args):
         publish.assert_called_once_with(
             topic='update.request.testing', msg=mock.ANY)
 
-        app = TestApp(main({}, testing=u'lloyd', session=self.db, **self.app_settings))
+        with mock.patch('bodhi.server.Session.remove'):
+            app = TestApp(main({}, testing=u'lloyd', session=self.db, **self.app_settings))
         update = self.get_update(nvr)
         update['csrf_token'] = app.get('/csrf').json_body['csrf_token']
         update['notes'] = u'testing!!!'
@@ -1004,7 +1010,8 @@ def test_provenpackager_request_privs(self, publish, *args):
         group2 = self.db.query(Group).filter_by(name=u'packager').one()
         user2.groups.append(group2)
 
-        app = TestApp(main({}, testing=u'ralph', session=self.db, **self.app_settings))
+        with mock.patch('bodhi.server.Session.remove'):
+            app = TestApp(main({}, testing=u'ralph', session=self.db, **self.app_settings))
         up_data = self.get_update(nvr)
         up_data['csrf_token'] = app.get('/csrf').json_body['csrf_token']
         res = app.post_json('/updates/', up_data)
@@ -1017,7 +1024,8 @@ def test_provenpackager_request_privs(self, publish, *args):
         self.assertEqual(build.update.request, UpdateRequest.testing)
 
         # Try and submit the update to stable as a non-provenpackager
-        app = TestApp(main({}, testing=u'ralph', session=self.db, **self.app_settings))
+        with mock.patch('bodhi.server.Session.remove'):
+            app = TestApp(main({}, testing=u'ralph', session=self.db, **self.app_settings))
         post_data = dict(update=nvr, request='stable',
                          csrf_token=app.get('/csrf').json_body['csrf_token'])
         res = app.post_json('/updates/%s/request' % str(nvr), post_data, status=400)
@@ -1056,7 +1064,9 @@ def test_provenpackager_request_privs(self, publish, *args):
         update.request = UpdateRequest.testing
 
         # Try and submit the update to stable as a proventester
-        app = TestApp(main({}, testing=u'bob', session=self.db, **self.app_settings))
+        with mock.patch('bodhi.server.Session.remove'):
+            app = TestApp(main({}, testing=u'bob', session=self.db, **self.app_settings))
+
         res = app.post_json('/updates/%s/request' % str(nvr),
                             dict(update=nvr, request='stable',
                                  csrf_token=app.get('/csrf').json_body['csrf_token']),
@@ -1064,7 +1074,9 @@ def test_provenpackager_request_privs(self, publish, *args):
 
         self.assertEqual(res.json_body['update']['request'], 'stable')
 
-        app = TestApp(main({}, testing=u'bob', session=self.db, **self.app_settings))
+        with mock.patch('bodhi.server.Session.remove'):
+            app = TestApp(main({}, testing=u'bob', session=self.db, **self.app_settings))
+
         res = app.post_json('/updates/%s/request' % str(nvr),
                             dict(update=nvr, request='obsolete',
                                  csrf_token=app.get('/csrf').json_body['csrf_token']),
@@ -1078,18 +1090,24 @@ def test_provenpackager_request_privs(self, publish, *args):
         self.assertEqual(update.status, UpdateStatus.obsolete)
 
         # Test that bob has can_edit True, provenpackager
-        app = TestApp(main({}, testing=u'bob', session=self.db, **self.app_settings))
+        with mock.patch('bodhi.server.Session.remove'):
+            app = TestApp(main({}, testing=u'bob', session=self.db, **self.app_settings))
+
         res = app.get('/updates/%s' % str(nvr), status=200)
         self.assertEqual(res.json_body['can_edit'], True)
 
         # Test that ralph has can_edit True, they submitted it.
-        app = TestApp(main({}, testing=u'ralph', session=self.db, **self.app_settings))
+        with mock.patch('bodhi.server.Session.remove'):
+            app = TestApp(main({}, testing=u'ralph', session=self.db, **self.app_settings))
+
         res = app.get('/updates/%s' % str(nvr), status=200)
         self.assertEqual(res.json_body['can_edit'], True)
 
         # Test that someuser has can_edit False, they are unrelated
         # This check *failed* with the old acls code.
-        app = TestApp(main({}, testing=u'someuser', session=self.db, **self.app_settings))
+        with mock.patch('bodhi.server.Session.remove'):
+            app = TestApp(main({}, testing=u'someuser', session=self.db, **self.app_settings))
+
         res = app.get('/updates/%s' % str(nvr), status=200)
         self.assertEqual(res.json_body['can_edit'], False)
 
@@ -1101,7 +1119,9 @@ def test_provenpackager_request_privs(self, publish, *args):
             'authtkt.secure': True,
         })
 
-        app = TestApp(main({}, session=self.db, **anonymous_settings))
+        with mock.patch('bodhi.server.Session.remove'):
+            app = TestApp(main({}, session=self.db, **anonymous_settings))
+
         res = app.get('/updates/%s' % str(nvr), status=200)
         self.assertEqual(res.json_body['can_edit'], False)
 
@@ -4350,10 +4370,9 @@ def test_push_to_batched_button_not_present_when_test_gating_status_failed(self,
         update.test_gating_status = TestGatingStatus.failed
         update.comment(self.db, 'works', 1, 'bowlofeggs')
         self.db.commit()
-        self.app_settings['test_gating.required'] = True
-        app = TestApp(main({}, testing=u'guest', **self.app_settings))
+        self.app.app.registry.settings['test_gating.required'] = True
 
-        resp = app.get('/updates/%s' % nvr, headers={'Accept': 'text/html'})
+        resp = self.app.get('/updates/%s' % nvr, headers={'Accept': 'text/html'})
 
         self.assertNotIn('Push to Batched', resp)
         self.assertNotIn('Push to Stable', resp)
@@ -4376,10 +4395,9 @@ def test_push_to_batched_button_present_when_test_gating_status_passed(self, pub
         update.test_gating_status = TestGatingStatus.passed
         update.comment(self.db, 'works', 1, 'bowlofeggs')
         self.db.commit()
-        self.app_settings['test_gating.required'] = True
-        app = TestApp(main({}, testing=u'guest', **self.app_settings))
+        self.app.app.registry.settings['test_gating.required'] = True
 
-        resp = app.get('/updates/%s' % nvr, headers={'Accept': 'text/html'})
+        resp = self.app.get('/updates/%s' % nvr, headers={'Accept': 'text/html'})
 
         self.assertIn('Push to Batched', resp)
         self.assertNotIn('Push to Stable', resp)
diff --git a/bodhi/tests/server/test___init__.py b/bodhi/tests/server/test___init__.py
--- a/bodhi/tests/server/test___init__.py
+++ b/bodhi/tests/server/test___init__.py
@@ -220,6 +220,13 @@ def test_authtkt_timeout_undefined(self, set_authorization_policy, set_authentic
         self.assertTrue(isinstance(policy, authorization.ACLAuthorizationPolicy))
         set_authorization_policy.assert_called_once_with(policy)
 
+    def test_calls_session_remove(self):
+        """Let's assert that main() calls Session.remove()."""
+        with mock.patch('bodhi.server.Session.remove') as remove:
+            server.main({}, **self.app_settings)
+
+        remove.assert_called_once_with()
+
     @mock.patch('bodhi.server.bugs.set_bugtracker')
     def test_calls_set_bugtracker(self, set_bugtracker):
         """
@@ -271,42 +278,3 @@ def test_warms_up_releases_cache(self):
 
         # The cache should have a release in it now - let's just spot check it
         self.assertEqual(models.Release._all_releases['current'][0]['name'], 'F17')
-
-
-class TestGetDbSessionForRequest(unittest.TestCase):
-    """Test the get_db_session_for_request() function."""
-
-    def test_cleanup_exception(self):
-        """Test cleanup() when there is an Exception."""
-        request = mock.Mock()
-        session = server.get_db_session_for_request(request)
-        cleanup = request.add_finished_callback.mock_calls[0][1][0]
-        request.exception = IOError('The Internet ran out of cats.')
-
-        cleanup(request)
-
-        # Since there was an Exception, the session should have been rolled back and closed.
-        self.assertEqual(session.rollback.mock_calls, [mock.call()])
-        self.assertEqual(session.commit.mock_calls, [])
-        self.assertEqual(session.close.mock_calls, [mock.call()])
-
-    def test_cleanup_no_exception(self):
-        """Test cleanup() when there is not an Exception."""
-        request = mock.Mock()
-        session = server.get_db_session_for_request(request)
-        cleanup = request.add_finished_callback.mock_calls[0][1][0]
-        request.exception = None
-
-        cleanup(request)
-
-        # Since there was no Exception, the session should have been committed and closed.
-        self.assertEqual(session.rollback.mock_calls, [])
-        self.assertEqual(session.commit.mock_calls, [mock.call()])
-        self.assertEqual(session.close.mock_calls, [mock.call()])
-
-    def test_session_from_registry_sessionmaker(self):
-        """Assert the session is created using the sessionmaker in the registry."""
-        mock_request = mock.Mock()
-        session = server.get_db_session_for_request(mock_request)
-        mock_request.registry.sessionmaker.assert_called_once_with()
-        self.assertEqual(session, mock_request.registry.sessionmaker.return_value)
diff --git a/bodhi/tests/server/test_metadata.py b/bodhi/tests/server/test_metadata.py
--- a/bodhi/tests/server/test_metadata.py
+++ b/bodhi/tests/server/test_metadata.py
@@ -261,12 +261,11 @@ def setUp(self):
         }]
 
     def tearDown(self):
-        shutil.rmtree(self.tempdir)
         config['mash_stage_dir'] = self._mash_stage_dir
         config['mash_dir'] = self._mash_dir
         config['cache_dir'] = None
         shutil.rmtree(self._new_mash_stage_dir)
-        super(TestUpdateInfoMetadata, self).setUp()
+        super(TestUpdateInfoMetadata, self).tearDown()
 
     def _verify_updateinfo(self, repodata):
         updateinfos = glob.glob(join(repodata, "*-updateinfo.xml*"))
diff --git a/bodhi/tests/server/test_renderers.py b/bodhi/tests/server/test_renderers.py
--- a/bodhi/tests/server/test_renderers.py
+++ b/bodhi/tests/server/test_renderers.py
@@ -19,6 +19,7 @@
 
 import copy
 import datetime
+import mock
 import re
 import unittest
 
@@ -52,7 +53,8 @@ def test_renderer_jpeg(self):
             'captcha.padding': '5',
             'captcha.ttl': '300',
         })
-        app = TestApp(main({}, session=self.db, **settings))
+        with mock.patch('bodhi.server.Session.remove'):
+            app = TestApp(main({}, session=self.db, **settings))
 
         res = app.get('/updates/FEDORA-{}-a3bbe1a8f2'.format(datetime.datetime.utcnow().year),
                       status=200,
diff --git a/bodhi/tests/server/test_webapp.py b/bodhi/tests/server/test_webapp.py
new file mode 100644
--- /dev/null
+++ b/bodhi/tests/server/test_webapp.py
@@ -0,0 +1,68 @@
+# -*- coding: utf-8 -*-
+# Copyright © 2018 Red Hat, Inc.
+#
+# This file is part of Bodhi.
+#
+# This program is free software; you can redistribute it and/or
+# modify it under the terms of the GNU General Public License
+# as published by the Free Software Foundation; either version 2
+# of the License, or (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
+"""Tests for bodhi.server.webapp."""
+
+import mock
+import unittest
+
+from bodhi.server import webapp
+
+
+class TestCompleteDatabaseSession(unittest.TestCase):
+    """Test the _complete_database_session() function."""
+
+    def test_cleanup_exception(self):
+        """Test for rollback() when there is an Exception."""
+        request = mock.Mock()
+        request.exception = IOError('The Internet ran out of cats.')
+
+        with mock.patch('bodhi.server.Session') as Session_mock:
+            webapp._complete_database_session(request)
+
+        # Since there was an Exception, the session should have been rolled back and closed.
+        self.assertEqual(Session_mock.return_value.rollback.mock_calls, [mock.call()])
+        self.assertEqual(Session_mock.return_value.commit.mock_calls, [])
+        self.assertEqual(Session_mock.return_value.close.mock_calls, [mock.call()])
+        self.assertEqual(Session_mock.remove.mock_calls, [mock.call()])
+
+    def test_cleanup_no_exception(self):
+        """Test cleanup() when there is not an Exception."""
+        request = mock.Mock()
+        request.exception = None
+
+        with mock.patch('bodhi.server.Session') as Session:
+            webapp._complete_database_session(request)
+
+        # Since there was no Exception, the session should have been committed and closed.
+        self.assertEqual(Session.return_value.rollback.mock_calls, [])
+        self.assertEqual(Session.return_value.commit.mock_calls, [mock.call()])
+        self.assertEqual(Session.return_value.close.mock_calls, [mock.call()])
+        self.assertEqual(Session.remove.mock_calls, [mock.call()])
+
+
+class TestPrepareRequest(unittest.TestCase):
+    """Test the _prepare_request() function."""
+
+    def test_adds_db_cleanup_callback(self):
+        event = mock.MagicMock()
+
+        webapp._prepare_request(event)
+
+        event.request.add_finished_callback.assert_called_once_with(
+            webapp._complete_database_session)