Repository: fedora-infra/bodhi

Issue:
bodhi.server.util.call_api should use connection pooling
The ```bodhi.server.util.call_api()``` function should use requests connection pooling via a Session object. This will make repeated calls to it more efficient.

Thanks to @jeremycline for the suggestion!


Discussion:

Resolving patch:
diff --git a/bodhi/server/util.py b/bodhi/server/util.py
--- a/bodhi/server/util.py
+++ b/bodhi/server/util.py
@@ -48,6 +48,8 @@
 
 _ = TranslationStringFactory('bodhi')
 
+http_session = requests.Session()
+
 
 def header(x):
     """Display a given message as a heading."""
@@ -1162,15 +1164,15 @@ def call_api(api_url, service_name, error_key=None, method='GET', data=None):
         base_error_msg = (
             'Bodhi failed to send POST request to {0} at the following URL '
             '"{1}". The status code was "{2}".')
-        rv = requests.post(api_url,
-                           headers={'Content-Type': 'application/json'},
-                           data=json.dumps(data),
-                           timeout=60)
+        rv = http_session.post(api_url,
+                               headers={'Content-Type': 'application/json'},
+                               data=json.dumps(data),
+                               timeout=60)
     else:
         base_error_msg = (
             'Bodhi failed to get a resource from {0} at the following URL '
             '"{1}". The status code was "{2}".')
-        rv = requests.get(api_url, timeout=60)
+        rv = http_session.get(api_url, timeout=60)
     if rv.status_code == 200:
         return rv.json()
     elif rv.status_code == 500:


Test patch:
diff --git a/bodhi/tests/server/test_models.py b/bodhi/tests/server/test_models.py
--- a/bodhi/tests/server/test_models.py
+++ b/bodhi/tests/server/test_models.py
@@ -409,8 +409,8 @@ def test_same_build_types(self):
         # This should not raise any Exception.
         self.db.flush()
 
-    @mock.patch('bodhi.server.util.requests.get')
-    def test_get_pkg_committers_from_pagure(self, mock_get):
+    @mock.patch('bodhi.server.util.http_session')
+    def test_get_pkg_committers_from_pagure(self, session):
         """ Ensure that the package committers can be found using the Pagure
         API.
         """
@@ -445,13 +445,13 @@ def test_get_pkg_committers_from_pagure(self, mock_get):
                 "name": "mprahl"
             }
         }
-        mock_get.return_value.json.return_value = json_output
-        mock_get.return_value.status_code = 200
+        session.get.return_value.json.return_value = json_output
+        session.get.return_value.status_code = 200
         rv = self.package.get_pkg_committers_from_pagure()
         assert rv == (['mprahl'], ['factory2']), rv
 
-    @mock.patch('bodhi.server.util.requests.get')
-    def test_get_pkg_committers_container_from_pagure(self, mock_get):
+    @mock.patch('bodhi.server.util.http_session')
+    def test_get_pkg_committers_container_from_pagure(self, session):
         """ Ensure that the container committers can be found using the Pagure
         API.
         """
@@ -484,8 +484,8 @@ def test_get_pkg_committers_container_from_pagure(self, mock_get):
                 "name": "mprahl"
             }
         }
-        mock_get.return_value.json.return_value = json_output
-        mock_get.return_value.status_code = 200
+        session.get.return_value.json.return_value = json_output
+        session.get.return_value.status_code = 200
         # Even though Bodhi doesn't support containers yet, let's mock this
         # package to have the type set to `container` to make sure the code in
         # get_pkg_committers_from_pagure works with containers in the future.
@@ -983,17 +983,17 @@ def test_modify_bugs_stable_no_close(self, comment, close):
         # No bugs should have been closed
         self.assertEqual(close.call_count, 0)
 
-    @mock.patch('bodhi.server.util.requests.get')
+    @mock.patch('bodhi.server.util.http_session')
     @mock.patch.dict(util.config, {
         'critpath.type': 'pdc',
         'pdc_url': 'http://domain.local'
     })
-    def test_contains_critpath_component(self, mock_get):
+    def test_contains_critpath_component(self, session):
         """ Verifies that the static function of contains_critpath_component
         determines that one of the builds has a critpath component.
         """
-        mock_get.return_value.status_code = 200
-        mock_get.return_value.json.return_value = {
+        session.get.return_value.status_code = 200
+        session.get.return_value.json.return_value = {
             'count': 2,
             'next': None,
             'previous': None,
@@ -1022,17 +1022,17 @@ def test_contains_critpath_component(self, mock_get):
         self.assertTrue(update.contains_critpath_component(
             update.builds, update.release.name))
 
-    @mock.patch('bodhi.server.util.requests.get')
+    @mock.patch('bodhi.server.util.http_session')
     @mock.patch.dict(util.config, {
         'critpath.type': 'pdc',
         'pdc_url': 'http://domain.local'
     })
-    def test_contains_critpath_component_not_critpath(self, mock_get):
+    def test_contains_critpath_component_not_critpath(self, session):
         """ Verifies that the static function of contains_critpath_component
         determines that none of the builds are critpath components.
         """
-        mock_get.return_value.status_code = 200
-        mock_get.return_value.json.return_value = {
+        session.get.return_value.status_code = 200
+        session.get.return_value.json.return_value = {
             'count': 2,
             'next': None,
             'previous': None,
diff --git a/bodhi/tests/server/test_util.py b/bodhi/tests/server/test_util.py
--- a/bodhi/tests/server/test_util.py
+++ b/bodhi/tests/server/test_util.py
@@ -144,17 +144,17 @@ def test_get_critpath_components_pkgdb_success(self, mock_get_critpath):
         pkgs = util.get_critpath_components('f20')
         assert critpath_pkgs == pkgs, pkgs
 
-    @mock.patch('bodhi.server.util.requests.get')
+    @mock.patch('bodhi.server.util.http_session')
     @mock.patch.dict(util.config, {
         'critpath.type': 'pdc',
         'pdc_url': 'http://domain.local'
     })
-    def test_get_critpath_components_pdc_error(self, mock_get):
+    def test_get_critpath_components_pdc_error(self, session):
         """ Ensure an error is thrown in Bodhi if there is an error in PDC
         getting the critpath packages.
         """
-        mock_get.return_value.status_code = 500
-        mock_get.return_value.json.return_value = \
+        session.get.return_value.status_code = 500
+        session.get.return_value.json.return_value = \
             {'error': 'some error'}
         try:
             util.get_critpath_components('f25')
@@ -180,19 +180,19 @@ def test_get_critpath_components_not_pdc_not_rpm(self, mock_log):
                    'for non-RPM components')
         mock_log.warning.assert_called_once_with(warning)
 
-    @mock.patch('bodhi.server.util.requests.get')
+    @mock.patch('bodhi.server.util.http_session')
     @mock.patch.dict(util.config, {
         'critpath.type': 'pdc',
         'pdc_url': 'http://domain.local'
     })
-    def test_get_critpath_components_pdc_success(self, mock_get):
+    def test_get_critpath_components_pdc_success(self, session):
         """ Ensure that critpath packages can be found using PDC.
         """
         pdc_url = \
             'http://domain.local/rest_api/v1/component-branches/?page_size=1'
         pdc_next_url = '{0}&page=2'.format(pdc_url)
-        mock_get.return_value.status_code = 200
-        mock_get.return_value.json.side_effect = [
+        session.get.return_value.status_code = 200
+        session.get.return_value.json.side_effect = [
             {
                 'count': 2,
                 'next': pdc_next_url,
@@ -232,17 +232,17 @@ def test_get_critpath_components_pdc_success(self, mock_get):
         # We can't verify all the calls made because the URL GET parameters
         # in the URL may have different orders based on the system/Python
         # version.
-        mock_get.assert_called_with(pdc_next_url, timeout=60)
+        session.get.assert_called_with(pdc_next_url, timeout=60)
         # Verify there were two GET requests made and two .json() calls
-        assert mock_get.call_count == 2, mock_get.call_count
-        assert mock_get.return_value.json.call_count == 2, \
-            mock_get.return_value.json.call_count
+        assert session.get.call_count == 2, session.get.call_count
+        assert session.get.return_value.json.call_count == 2, \
+            session.get.return_value.json.call_count
 
-    @mock.patch('bodhi.server.util.requests.get')
-    def test_pagure_api_get(self, mock_get):
+    @mock.patch('bodhi.server.util.http_session')
+    def test_pagure_api_get(self, session):
         """ Ensure that an API request to Pagure works as expected.
         """
-        mock_get.return_value.status_code = 200
+        session.get.return_value.status_code = 200
         expected_json = {
             "access_groups": {
                 "admin": [],
@@ -274,17 +274,17 @@ def test_pagure_api_get(self, mock_get):
                 "name": "mprahl"
             }
         }
-        mock_get.return_value.json.return_value = expected_json
+        session.get.return_value.json.return_value = expected_json
         rv = util.pagure_api_get('http://domain.local/api/0/rpms/python')
         assert rv == expected_json, rv
 
-    @mock.patch('bodhi.server.util.requests.get')
-    def test_pagure_api_get_non_500_error(self, mock_get):
+    @mock.patch('bodhi.server.util.http_session')
+    def test_pagure_api_get_non_500_error(self, session):
         """ Ensure that an API request to Pagure that raises an error that is
         not a 500 error returns the actual error message from the JSON.
         """
-        mock_get.return_value.status_code = 404
-        mock_get.return_value.json.return_value = {
+        session.get.return_value.status_code = 404
+        session.get.return_value.json.return_value = {
             "error": "Project not found",
             "error_code": "ENOPROJECT"
         }
@@ -300,12 +300,12 @@ def test_pagure_api_get_non_500_error(self, mock_get):
             '"404". The error was "Project not found".')
         assert actual_error == expected_error, actual_error
 
-    @mock.patch('bodhi.server.util.requests.get')
-    def test_pagure_api_get_500_error(self, mock_get):
+    @mock.patch('bodhi.server.util.http_session')
+    def test_pagure_api_get_500_error(self, session):
         """ Ensure that an API request to Pagure that triggers a 500 error
         raises the expected error message.
         """
-        mock_get.return_value.status_code = 500
+        session.get.return_value.status_code = 500
         try:
             util.pagure_api_get('http://domain.local/api/0/rpms/python')
             assert False, 'Did not raise a RuntimeError'
@@ -318,13 +318,13 @@ def test_pagure_api_get_500_error(self, mock_get):
             '"500".')
         assert actual_error == expected_error, actual_error
 
-    @mock.patch('bodhi.server.util.requests.get')
-    def test_pagure_api_get_non_500_error_no_json(self, mock_get):
+    @mock.patch('bodhi.server.util.http_session')
+    def test_pagure_api_get_non_500_error_no_json(self, session):
         """ Ensure that an API request to Pagure that raises an error that is
         not a 500 error and has no JSON returns an error.
         """
-        mock_get.return_value.status_code = 404
-        mock_get.return_value.json.side_effect = ValueError('Not JSON')
+        session.get.return_value.status_code = 404
+        session.get.return_value.json.side_effect = ValueError('Not JSON')
         try:
             util.pagure_api_get('http://domain.local/api/0/rpms/python')
             assert False, 'Did not raise a RuntimeError'
@@ -337,11 +337,11 @@ def test_pagure_api_get_non_500_error_no_json(self, mock_get):
             '"404". The error was "".')
         assert actual_error == expected_error, actual_error
 
-    @mock.patch('bodhi.server.util.requests.get')
-    def test_pdc_api_get(self, mock_get):
+    @mock.patch('bodhi.server.util.http_session')
+    def test_pdc_api_get(self, session):
         """ Ensure that an API request to PDC works as expected.
         """
-        mock_get.return_value.status_code = 200
+        session.get.return_value.status_code = 200
         expected_json = {
             "count": 1,
             "next": None,
@@ -362,17 +362,17 @@ def test_pdc_api_get(self, mock_get):
                 }
             ]
         }
-        mock_get.return_value.json.return_value = expected_json
+        session.get.return_value.json.return_value = expected_json
         rv = util.pdc_api_get(
             'http://domain.local/rest_api/v1/component-branch-slas/')
         assert rv == expected_json, rv
 
-    @mock.patch('bodhi.server.util.requests.get')
-    def test_pdc_api_get_500_error(self, mock_get):
+    @mock.patch('bodhi.server.util.http_session')
+    def test_pdc_api_get_500_error(self, session):
         """ Ensure that an API request to PDC that triggers a 500 error
         raises the expected error message.
         """
-        mock_get.return_value.status_code = 500
+        session.get.return_value.status_code = 500
         try:
             util.pdc_api_get(
                 'http://domain.local/rest_api/v1/component-branch-slas/')
@@ -386,13 +386,13 @@ def test_pdc_api_get_500_error(self, mock_get):
             'status code was "500".')
         assert actual_error == expected_error, actual_error
 
-    @mock.patch('bodhi.server.util.requests.get')
-    def test_pdc_api_get_non_500_error(self, mock_get):
+    @mock.patch('bodhi.server.util.http_session')
+    def test_pdc_api_get_non_500_error(self, session):
         """ Ensure that an API request to PDC that raises an error that is
         not a 500 error returns the returned JSON.
         """
-        mock_get.return_value.status_code = 404
-        mock_get.return_value.json.return_value = {
+        session.get.return_value.status_code = 404
+        session.get.return_value.json.return_value = {
             "detail": "Not found."
         }
         try:
@@ -409,13 +409,13 @@ def test_pdc_api_get_non_500_error(self, mock_get):
             '"{\'detail\': \'Not found.\'}".')
         assert actual_error == expected_error, actual_error
 
-    @mock.patch('bodhi.server.util.requests.get')
-    def test_pdc_api_get_non_500_error_no_json(self, mock_get):
+    @mock.patch('bodhi.server.util.http_session')
+    def test_pdc_api_get_non_500_error_no_json(self, session):
         """ Ensure that an API request to PDC that raises an error that is
         not a 500 error and has no JSON returns an error.
         """
-        mock_get.return_value.status_code = 404
-        mock_get.return_value.json.side_effect = ValueError('Not JSON')
+        session.get.return_value.status_code = 404
+        session.get.return_value.json.side_effect = ValueError('Not JSON')
         try:
             util.pdc_api_get(
                 'http://domain.local/rest_api/v1/component-branch-slas/3/')
@@ -437,18 +437,18 @@ def test_get_nvr(self):
         for element in result:
             assert isinstance(element, unicode)
 
-    @mock.patch('bodhi.server.util.requests.post')
-    def test_greenwave_api_post(self, mock_post):
+    @mock.patch('bodhi.server.util.http_session')
+    def test_greenwave_api_post(self, session):
         """ Ensure that a POST request to Greenwave works as expected.
         """
-        mock_post.return_value.status_code = 200
+        session.post.return_value.status_code = 200
         expected_json = {
             'policies_satisified': True,
             'summary': 'All tests passed',
             'applicable_policies': ['taskotron_release_critical_tasks'],
             'unsatisfied_requirements': []
         }
-        mock_post.return_value.json.return_value = expected_json
+        session.post.return_value.json.return_value = expected_json
         data = {
             'product_version': 'fedora-26',
             'decision_context': 'bodhi_push_update_stable',
@@ -458,12 +458,12 @@ def test_greenwave_api_post(self, mock_post):
                                            data)
         assert decision == expected_json, decision
 
-    @mock.patch('bodhi.server.util.requests.post')
-    def test_greenwave_api_post_500_error(self, mock_post):
+    @mock.patch('bodhi.server.util.http_session')
+    def test_greenwave_api_post_500_error(self, session):
         """ Ensure that a POST request to Greenwave that triggers a 500 error
         raises the expected error message.
         """
-        mock_post.return_value.status_code = 500
+        session.post.return_value.status_code = 500
         try:
             data = {
                 'product_version': 'fedora-26',
@@ -481,13 +481,13 @@ def test_greenwave_api_post_500_error(self, mock_post):
             '"http://domain.local/api/v1.0/decision". The status code was "500".')
         assert actual_error == expected_error, actual_error
 
-    @mock.patch('bodhi.server.util.requests.post')
-    def test_greenwave_api_post_non_500_error(self, mock_post):
+    @mock.patch('bodhi.server.util.http_session')
+    def test_greenwave_api_post_non_500_error(self, session):
         """ Ensure that a POST request to Greenwave that raises an error that is
         not a 500 error returns the returned JSON.
         """
-        mock_post.return_value.status_code = 404
-        mock_post.return_value.json.return_value = {
+        session.post.return_value.status_code = 404
+        session.post.return_value.json.return_value = {
             "message": "Not found."
         }
         try:
@@ -508,13 +508,13 @@ def test_greenwave_api_post_non_500_error(self, mock_post):
             'The error was "{\'message\': \'Not found.\'}".')
         assert actual_error == expected_error, actual_error
 
-    @mock.patch('bodhi.server.util.requests.post')
-    def test_greenwave_api_post_non_500_error_no_json(self, mock_post):
+    @mock.patch('bodhi.server.util.http_session')
+    def test_greenwave_api_post_non_500_error_no_json(self, session):
         """ Ensure that a POST request to Greenwave that raises an error that is
         not a 500 error and has no JSON returns an error.
         """
-        mock_post.return_value.status_code = 404
-        mock_post.return_value.json.side_effect = ValueError('Not JSON')
+        session.post.return_value.status_code = 404
+        session.post.return_value.json.side_effect = ValueError('Not JSON')
         try:
             data = {
                 'product_version': 'fedora-26',