Repository: fedora-infra/bodhi

Issue:
PDC queries take too long and cause Bodhi requests to time out
[```get_critpath_components_from_pdc()```](https://github.com/fedora-infra/bodhi/blob/3.0.0/bodhi/server/util.py#L767-L798) is extremely slow. During some testing, @puiterwijk found that individual requests to PDC for f27 take 8-30 seconds, and there are 8 pages/8 requests. This is why the proxy server is timing out on Bodhi requests.


Discussion:
The lack of critpath data for F27 is a critical issue that's causing us (QA) a serious problem I can't get around: openQA is running almost no update tests for F27 updates, because none are marked as critpath. Only updates to packages on the openQA scheduler's own testing 'whitelist' (which is mainly FreeIPA-related packages) are being tested at present.

Resolving patch:
diff --git a/bodhi/server/models.py b/bodhi/server/models.py
--- a/bodhi/server/models.py
+++ b/bodhi/server/models.py
@@ -1705,12 +1705,16 @@ def contains_critpath_component(builds, release_name):
             bool: ``True`` if the update contains a critical path package, ``False`` otherwise.
         component
         """
+        relname = release_name.lower()
+        components = defaultdict(list)
+        # Get the mess down to a dict of ptype -> [pname]
         for build in builds:
-            # This function call is cached, so there is no need to optimize
-            # it here.
-            critpath_components = get_critpath_components(
-                release_name.lower(), build.package.type.value)
-            if build.package.name in critpath_components:
+            ptype = build.package.type.value
+            pname = build.package.name
+            components[ptype].append(pname)
+
+        for ptype in components:
+            if get_critpath_components(relname, ptype, frozenset(components[ptype])):
                 return True
 
         return False
diff --git a/bodhi/server/util.py b/bodhi/server/util.py
--- a/bodhi/server/util.py
+++ b/bodhi/server/util.py
@@ -29,6 +29,7 @@
 import socket
 import subprocess
 import tempfile
+import time
 import urllib
 
 from kitchen.iterutils import iterate
@@ -243,14 +244,16 @@ def __get__(self, obj, objtype):
 
 
 @memoized
-def get_critpath_components(collection='master', component_type='rpm'):
+def get_critpath_components(collection='master', component_type='rpm', components=None):
     """
-    Return a list of critical path packages for a given collection.
+    Return a list of critical path packages for a given collection, filtered by components.
 
     Args:
-    collection (basestring): The collection/branch to search. Defaults to 'master'.
-    component_type (basestring): The component type to search for. This only affects PDC queries.
-        Defaults to 'rpm'.
+        collection (basestring): The collection/branch to search. Defaults to 'master'.
+        component_type (basestring): The component type to search for. This only affects PDC
+            queries. Defaults to 'rpm'.
+        components (frozenset or None): The list of components we are interested in. If None (the
+            default), all components for the given collection and type are returned.
     Returns:
         list: The critpath components for the given collection and type.
     """
@@ -268,9 +271,15 @@ def get_critpath_components(collection='master', component_type='rpm'):
             critpath_components = results['pkgs'][collection]
     elif critpath_type == 'pdc':
         critpath_components = get_critpath_components_from_pdc(
-            collection, component_type)
+            collection, component_type, components)
     else:
         critpath_components = config.get('critpath_pkgs')
+
+    # Filter the list of components down to what was requested, in case the specific path did
+    # not take our request into account.
+    if components is not None:
+        critpath_components = [c for c in critpath_components if c in components]
+
     return critpath_components
 
 
@@ -1133,43 +1142,63 @@ def sort_severity(value):
     return value_map.get(value, 99)
 
 
-def get_critpath_components_from_pdc(branch, component_type='rpm'):
+# If we need to know about more components than this constant, we will just get the full
+# list, rather than a query per package. This is because at some point, just going through
+# paging becomes more performant than getting the page for every component.
+PDC_CRITPATH_COMPONENTS_GETALL_LIMIT = 10
+
+
+def get_critpath_components_from_pdc(branch, component_type='rpm', components=None):
     """
     Search PDC for critical path packages based on the specified branch.
 
     Args:
         branch (basestring): The branch name to search by.
         component_type (basestring): The component type to search by. Defaults to ``rpm``.
+        components (frozenset or None): The list of components we are interested in. If None (the
+            default), all components for the given branch and type are returned.
     Returns:
         list: Critical path package names.
     """
     pdc_api_url = '{}/rest_api/v1/component-branches/'.format(
         config.get('pdc_url').rstrip('/'))
-    query_args = urllib.urlencode({
+    query_args = {
         'active': 'true',
         'critical_path': 'true',
         'name': branch,
         'page_size': 100,
         'type': component_type
-    })
-    pdc_api_url_with_args = '{0}?{1}'.format(pdc_api_url, query_args)
+    }
 
     critpath_pkgs_set = set()
-    while True:
-        pdc_request_json = pdc_api_get(pdc_api_url_with_args)
+    if components and len(components) < PDC_CRITPATH_COMPONENTS_GETALL_LIMIT:
+        # Do a query for every single component
+        for component in components:
+            query_args['global_component'] = component
+            pdc_api_url_with_args = '{0}?{1}'.format(pdc_api_url, urllib.urlencode(query_args))
+            pdc_request_json = pdc_api_get(pdc_api_url_with_args)
+            for branch_rv in pdc_request_json['results']:
+                critpath_pkgs_set.add(branch_rv['global_component'])
+            if pdc_request_json['next']:
+                raise Exception('We got paging when requesting a single component?!')
+    else:
+        pdc_api_url_with_args = '{0}?{1}'.format(pdc_api_url, urllib.urlencode(query_args))
+        while True:
+            pdc_request_json = pdc_api_get(pdc_api_url_with_args)
 
-        for branch_rv in pdc_request_json['results']:
-            critpath_pkgs_set.add(branch_rv['global_component'])
+            for branch_rv in pdc_request_json['results']:
+                critpath_pkgs_set.add(branch_rv['global_component'])
 
-        if pdc_request_json['next']:
-            pdc_api_url_with_args = pdc_request_json['next']
-        else:
-            # There are no more results to iterate through
-            break
+            if pdc_request_json['next']:
+                pdc_api_url_with_args = pdc_request_json['next']
+            else:
+                # There are no more results to iterate through
+                break
     return list(critpath_pkgs_set)
 
 
-def call_api(api_url, service_name, error_key=None, method='GET', data=None, headers=None):
+def call_api(api_url, service_name, error_key=None, method='GET', data=None, headers=None,
+             retries=0):
     """
     Perform an HTTP request with response type and error handling.
 
@@ -1181,6 +1210,8 @@ def call_api(api_url, service_name, error_key=None, method='GET', data=None, hea
             service. If this is set to None, the JSON response will be used as the error message.
         method (basestring): The HTTP method to use for the request. Defaults to ``GET``.
         data (dict): Query string parameters that will be sent along with the request to the server.
+        retries (int): The number of times to retry, each after a 1 second sleep, if we get a
+            non-200 HTTP code. Defaults to 3.
     Returns:
         dict: A dictionary representing the JSON response from the remote service.
     Raises:
@@ -1205,6 +1236,9 @@ def call_api(api_url, service_name, error_key=None, method='GET', data=None, hea
         rv = http_session.get(api_url, timeout=60)
     if rv.status_code == 200:
         return rv.json()
+    elif retries:
+        time.sleep(1)
+        return call_api(api_url, service_name, error_key, method, data, headers, retries - 1)
     elif rv.status_code == 500:
         # There will be no JSON with an error message here
         error_msg = base_error_msg.format(
@@ -1238,7 +1272,7 @@ def pagure_api_get(pagure_api_url):
     Raises:
         RuntimeError: If the server did not give us a 200 code.
     """
-    return call_api(pagure_api_url, service_name='Pagure', error_key='error')
+    return call_api(pagure_api_url, service_name='Pagure', error_key='error', retries=3)
 
 
 def pdc_api_get(pdc_api_url):
@@ -1254,7 +1288,7 @@ def pdc_api_get(pdc_api_url):
     """
     # There is no error_key specified because the error key is not consistent
     # based on the error message
-    return call_api(pdc_api_url, service_name='PDC')
+    return call_api(pdc_api_url, service_name='PDC', retries=3)
 
 
 def greenwave_api_post(greenwave_api_url, data):
@@ -1272,7 +1306,7 @@ def greenwave_api_post(greenwave_api_url, data):
     # There is no error_key specified because the error key is not consistent
     # based on the error message
     return call_api(greenwave_api_url, service_name='Greenwave', method='POST',
-                    data=data)
+                    data=data, retries=3)
 
 
 def waiverdb_api_post(waiverdb_api_url, data):


Test patch:
diff --git a/bodhi/tests/server/test_util.py b/bodhi/tests/server/test_util.py
--- a/bodhi/tests/server/test_util.py
+++ b/bodhi/tests/server/test_util.py
@@ -105,6 +105,53 @@ def test_short_true(self):
              "#1234567</a>"))
 
 
+@mock.patch('bodhi.server.util.time.sleep')
+class TestCallAPI(unittest.TestCase):
+    """Test the call_api() function."""
+
+    @mock.patch('bodhi.server.util.http_session.get')
+    def test_retries_failure(self, get, sleep):
+        """Assert correct operation of the retries argument when they never succeed."""
+        class FakeResponse(object):
+            def __init__(self, status_code):
+                self.status_code = status_code
+
+            def json(self):
+                return {'some': 'stuff'}
+
+        get.side_effect = [FakeResponse(503), FakeResponse(503)]
+
+        with self.assertRaises(RuntimeError) as exc:
+            util.call_api('url', 'service_name', retries=1)
+
+        self.assertEqual(
+            str(exc.exception),
+            ('Bodhi failed to get a resource from service_name at the following URL "url". The '
+             'status code was "503". The error was "{\'some\': \'stuff\'}".'))
+        self.assertEqual(get.mock_calls,
+                         [mock.call('url', timeout=60), mock.call('url', timeout=60)])
+        sleep.assert_called_once_with(1)
+
+    @mock.patch('bodhi.server.util.http_session.get')
+    def test_retries_success(self, get, sleep):
+        """Assert correct operation of the retries argument when they succeed eventually."""
+        class FakeResponse(object):
+            def __init__(self, status_code):
+                self.status_code = status_code
+
+            def json(self):
+                return {'some': 'stuff'}
+
+        get.side_effect = [FakeResponse(503), FakeResponse(200)]
+
+        res = util.call_api('url', 'service_name', retries=1)
+
+        self.assertEqual(res, {'some': 'stuff'})
+        self.assertEqual(get.mock_calls,
+                         [mock.call('url', timeout=60), mock.call('url', timeout=60)])
+        sleep.assert_called_once_with(1)
+
+
 class TestPushToBatchedOrStableButton(base.BaseTestCase):
     """Test the push_to_batched_or_stable_button() function."""
     def test_request_is_batched(self):
@@ -212,12 +259,12 @@ def test_config(self):
         assert config.get('sqlalchemy.url'), config
         assert config['sqlalchemy.url'], config
 
+    @mock.patch.dict(util.config, {'critpath.type': None, 'critpath_pkgs': ['kernel', 'glibc']})
     def test_get_critpath_components_dummy(self):
         """ Ensure that critpath packages can be found using the hardcoded
         list.
         """
-        pkgs = util.get_critpath_components()
-        assert 'kernel' in pkgs, pkgs
+        self.assertEqual(util.get_critpath_components(), ['kernel', 'glibc'])
 
     @mock.patch.object(pkgdb2client.PkgDB, 'get_critpath_packages')
     @mock.patch.dict(util.config, {
@@ -265,9 +312,7 @@ def test_get_critpath_components_pdc_error(self, session):
         assert 'The status code was "500".' in actual_error
 
     @mock.patch('bodhi.server.util.log')
-    @mock.patch.dict(util.config, {
-        'critpath.type': 'dummy',
-    })
+    @mock.patch.dict(util.config, {'critpath.type': None, 'critpath_pkgs': ['kernel', 'glibc']})
     def test_get_critpath_components_not_pdc_not_rpm(self, mock_log):
         """ Ensure a warning is logged when the critpath system is not pdc
         and the type of components to search for is not rpm.
@@ -278,6 +323,70 @@ def test_get_critpath_components_not_pdc_not_rpm(self, mock_log):
                    'for non-RPM components')
         mock_log.warning.assert_called_once_with(warning)
 
+    @mock.patch('bodhi.server.util.http_session')
+    @mock.patch.dict(util.config, {'critpath.type': 'pdc', 'pdc_url': 'http://domain.local'})
+    def test_get_critpath_components_pdc_paging_exception(self, session):
+        """Ensure that an Exception is raised if components are used and the response is paged."""
+        pdc_url = 'http://domain.local/rest_api/v1/component-branches/?page_size=1'
+        pdc_next_url = '{0}&page=2'.format(pdc_url)
+        session.get.return_value.status_code = 200
+        session.get.return_value.json.side_effect = [
+            {
+                'count': 2,
+                'next': pdc_next_url,
+                'previous': None,
+                'results': [
+                    {
+                        'active': True,
+                        'critical_path': True,
+                        'global_component': 'gcc',
+                        'id': 6,
+                        'name': 'f26',
+                        'slas': [],
+                        'type': 'rpm'
+                    }]}]
+
+        with self.assertRaises(Exception) as exc:
+            util.get_critpath_components('f26', 'rpm', frozenset(['gcc']))
+
+        self.assertEqual(str(exc.exception), 'We got paging when requesting a single component?!')
+        self.assertEqual(
+            session.get.mock_calls,
+            [mock.call(
+                ('http://domain.local/rest_api/v1/component-branches/?name=f26&global_component=gcc'
+                 '&page_size=100&critical_path=true&active=true&type=rpm'),
+                timeout=60),
+             mock.call().json()])
+
+    @mock.patch('bodhi.server.util.http_session')
+    @mock.patch.dict(util.config, {'critpath.type': 'pdc', 'pdc_url': 'http://domain.local'})
+    def test_get_critpath_pdc_with_components(self, session):
+        """Test the components argument to get_critpath_components()."""
+        session.get.return_value.status_code = 200
+        session.get.return_value.json.return_value = {
+            'count': 1,
+            'next': None,
+            'previous': None,
+            'results': [{
+                'active': True,
+                'critical_path': True,
+                'global_component': 'gcc',
+                'id': 6,
+                'name': 'f26',
+                'slas': [],
+                'type': 'rpm'}]}
+
+        pkgs = util.get_critpath_components('f26', 'rpm', frozenset(['gcc']))
+
+        self.assertEqual(pkgs, ['gcc'])
+        self.assertEqual(
+            session.get.mock_calls,
+            [mock.call(
+                ('http://domain.local/rest_api/v1/component-branches/?name=f26&global_component=gcc'
+                 '&page_size=100&critical_path=true&active=true&type=rpm'),
+                timeout=60),
+             mock.call().json()])
+
     @mock.patch('bodhi.server.util.http_session')
     @mock.patch.dict(util.config, {
         'critpath.type': 'pdc',