Repository: rucio/rucio

Issue:
Base client should retry when receiving a 502 error
Motivation
----------

ATLAS has observed a couple of times when requests to Rucio would fail with a 502 Bad Gateway error. This does not seem to be a wide-spread problem, but during those times it was happening consistently. Initial discussions pointed to HAProxy. 

Depending on which use of the Rucio client is failing, this can have detrimental effects on DPA. We could ask the users of the client to implement some protection around this, but there’s no doubt that it’s better for Rucio to handle it internally. The base client already has a mechanism for retrying requests that fail with a connection error.

Modification
------------

We should consider extending the retrying mechanism to also cover this type of error.



Discussion:
I would do the same for 503 and 504, but that's debatable.

Resolving patch:
diff --git a/lib/rucio/client/baseclient.py b/lib/rucio/client/baseclient.py
--- a/lib/rucio/client/baseclient.py
+++ b/lib/rucio/client/baseclient.py
@@ -34,7 +34,8 @@
 # - Andrew Lister <andrew.lister@stfc.ac.uk>, 2019
 # - Eli Chadwick <eli.chadwick@stfc.ac.uk>, 2020
 # - Patrick Austin <patrick.austin@stfc.ac.uk>, 2020
-# - Benedikt Ziemons <benedikt.ziemons@cern.ch>, 2020
+# - Benedikt Ziemons <benedikt.ziemons@cern.ch>, 2020-2021
+# - Radu Carpa <radu.carpa@cern.ch>, 2021
 
 '''
  Client class for callers of the Rucio system
@@ -114,6 +115,20 @@
     expiration_time=60,
 )
 
+STATUS_CODES_TO_RETRY = [502, 503, 504]
+MAX_RETRY_BACK_OFF_SECONDS = 10
+
+
+def back_off(retry_number, reason):
+    """
+    Sleep a certain amount of time which increases with the retry count
+    :param retry_number: the retry iteration
+    :param reason: the reason to backoff which will be shown to the user
+    """
+    sleep_time = min(MAX_RETRY_BACK_OFF_SECONDS, 0.25 * 2 ** retry_number)
+    LOG.warning("Waiting {}s due to reason: {} ".format(sleep_time, reason))
+    time.sleep(sleep_time)
+
 
 @REGION.cache_on_arguments(namespace='host_to_choose')
 def choice(hosts):
@@ -402,6 +417,9 @@ def _send_request(self, url, headers=None, type='GET', data=None, params=None, s
                     result = self.session.delete(url, headers=hds, data=data, verify=self.ca_cert, timeout=self.timeout)
                 else:
                     return
+                if result.status_code in STATUS_CODES_TO_RETRY:
+                    back_off(retry, reason='server returned {}'.format(result.status_code))
+                    continue
             except ConnectionError as error:
                 LOG.error('ConnectionError: ' + str(error))
                 if retry > self.request_retries:
@@ -437,7 +455,9 @@ def __get_token_userpass(self):
         for retry in range(self.AUTH_RETRIES + 1):
             try:
                 result = self.session.get(url, headers=headers, verify=self.ca_cert)
-                break
+                if result.status_code in STATUS_CODES_TO_RETRY:
+                    back_off(retry, reason='server returned {}'.format(result.status_code))
+                    continue
             except ConnectionError as error:
                 LOG.error('ConnectionError: ' + str(error))
                 if retry > self.request_retries:
@@ -521,6 +541,9 @@ def __refresh_token_OIDC(self):
                         LOG.debug("No new token was received, possibly invalid/expired \
                                    \ntoken or a token with no refresh token in Rucio DB")
                         return False
+                elif refresh_result.status_code in STATUS_CODES_TO_RETRY:
+                    back_off(retry, reason='server returned {}'.format(refresh_result.status_code))
+                    continue
                 else:
                     print("Rucio Client did not succeed to contact the \
                            \nRucio Auth Server when attempting token refresh.")
@@ -705,6 +728,9 @@ def __get_token_x509(self):
         for retry in range(self.AUTH_RETRIES + 1):
             try:
                 result = self.session.get(url, headers=headers, cert=cert, verify=self.ca_cert)
+                if result.status_code in STATUS_CODES_TO_RETRY:
+                    back_off(retry, reason='server returned {}'.format(result.status_code))
+                    continue
                 break
             except ConnectionError as error:
                 if 'alert certificate expired' in str(error):
@@ -750,6 +776,9 @@ def __get_token_ssh(self):
         for retry in range(self.AUTH_RETRIES + 1):
             try:
                 result = self.session.get(url, headers=headers, verify=self.ca_cert)
+                if result.status_code in STATUS_CODES_TO_RETRY:
+                    back_off(retry, reason='server returned {}'.format(result.status_code))
+                    continue
                 break
             except ConnectionError as error:
                 if 'alert certificate expired' in str(error):
@@ -783,6 +812,9 @@ def __get_token_ssh(self):
         for retry in range(self.AUTH_RETRIES + 1):
             try:
                 result = self.session.get(url, headers=headers, verify=self.ca_cert)
+                if result.status_code in STATUS_CODES_TO_RETRY:
+                    back_off(retry, reason='server returned {}'.format(result.status_code))
+                    continue
                 break
             except ConnectionError as error:
                 if 'alert certificate expired' in str(error):
@@ -821,6 +853,9 @@ def __get_token_gss(self):
             try:
                 result = self.session.get(url, headers=headers,
                                           verify=self.ca_cert, auth=HTTPKerberosAuth())
+                if result.status_code in STATUS_CODES_TO_RETRY:
+                    back_off(retry, reason='server returned {}'.format(result.status_code))
+                    continue
                 break
             except ConnectionError as error:
                 LOG.error('ConnectionError: ' + str(error))
@@ -855,6 +890,9 @@ def __get_token_saml(self):
         for retry in range(self.AUTH_RETRIES + 1):
             try:
                 SAML_auth_result = self.session.get(url, headers=headers)
+                if SAML_auth_result.status_code in STATUS_CODES_TO_RETRY:
+                    back_off(retry, reason='server returned {}'.format(SAML_auth_result.status_code))
+                    continue
 
                 if SAML_auth_result.headers['X-Rucio-Auth-Token']:
                     return SAML_auth_result.headers['X-Rucio-Auth-Token']


Test patch:
diff --git a/lib/rucio/tests/test_clients.py b/lib/rucio/tests/test_clients.py
--- a/lib/rucio/tests/test_clients.py
+++ b/lib/rucio/tests/test_clients.py
@@ -24,21 +24,69 @@
 # - Andrew Lister <andrew.lister@stfc.ac.uk>, 2019
 # - Patrick Austin <patrick.austin@stfc.ac.uk>, 2020
 # - Benedikt Ziemons <benedikt.ziemons@cern.ch>, 2020-2021
+# - Radu Carpa <radu.carpa@cern.ch>, 2021
 
 from __future__ import print_function
 
 import unittest
+from datetime import datetime, timedelta
+try:
+    from SimpleHTTPServer import SimpleHTTPRequestHandler
+except ImportError:
+    from http.server import SimpleHTTPRequestHandler
+try:
+    from SocketServer import TCPServer as HTTPServer
+except ImportError:
+    from http.server import HTTPServer
 from os import remove
+from threading import Thread
 
 import pytest
 
 from rucio.client.baseclient import BaseClient
 from rucio.client.client import Client
 from rucio.common.config import config_get, config_get_bool
-from rucio.common.exception import CannotAuthenticate, ClientProtocolNotSupported
+from rucio.common.exception import CannotAuthenticate, ClientProtocolNotSupported, RucioException
 from rucio.common.utils import get_tmp_dir
 
 
+class MockServer:
+    """
+    Start A simple http server in a separate thread to serve as MOCK for testing the client
+    """
+
+    class Handler(SimpleHTTPRequestHandler):
+        def send_code_and_message(self, code, headers, message):
+            """
+            Helper which wraps the quite-low-level BaseHTTPRequestHandler primitives and is used to send reponses.
+            """
+            self.send_response(code)
+            self.send_header("Content-type", "text/plain")
+            for name, content in headers.items():
+                self.send_header(name, content)
+            self.end_headers()
+            self.wfile.write(message.encode())
+
+    def __init__(self, request_handler_cls):
+        self.server = HTTPServer(('localhost', 0), request_handler_cls)
+        self.thread = Thread(target=self.server.serve_forever)
+        self.thread.daemon = True
+
+    def __enter__(self):
+        self.thread.start()
+        return self
+
+    def __exit__(self, exc_type, exc_val, exc_tb):
+        self.server.shutdown()
+        self.thread.join()
+        self.server.server_close()
+
+    @property
+    def base_url(self):
+        name, port = self.server.server_address
+        return 'http://{}:{}'.format(name, port)
+
+
 @pytest.mark.noparallel(reason='fails when run in parallel')
 class TestBaseClient(unittest.TestCase):
     """ To test Clients"""
@@ -99,6 +147,44 @@ def testClientProtocolNotSupported(self):
         with pytest.raises(ClientProtocolNotSupported):
             BaseClient(rucio_host='localhost', auth_host='junk://localhost', account='root', auth_type='userpass', creds=creds, **self.vo)
 
+    def testRetryOn502AlwaysFail(self):
+        """ CLIENTS (BASECLIENT): Ensure client retries on 502 error codes, but fails on repeated errors"""
+
+        class AlwaysFailWith502(MockServer.Handler):
+            def do_GET(self):
+                self.send_code_and_message(502, {}, '')
+
+        with MockServer(AlwaysFailWith502) as server:
+            with pytest.raises(CannotAuthenticate):
+                creds = {'username': 'ddmlab', 'password': 'secret'}
+                BaseClient(rucio_host=server.base_url, auth_host=server.base_url, account='root', auth_type='userpass', creds=creds, **self.vo)
+            with pytest.raises(RucioException):
+                creds = {'client_cert': self.usercert,
+                         'client_key': self.userkey}
+                BaseClient(rucio_host=server.base_url, auth_host=server.base_url, account='root', ca_cert=self.cacert, auth_type='x509', creds=creds, **self.vo)
+
+    def testRetryOn502SucceedsEventually(self):
+        """ CLIENTS (BASECLIENT): Ensure client retries on 502 error codes"""
+        invocations = []
+
+        class FailTwiceWith502(MockServer.Handler):
+            def do_GET(self, invocations=invocations):
+                invocations.append(self.path)
+                if len(invocations) <= 2:
+                    self.send_code_and_message(502, {}, '')
+                else:
+                    self.send_code_and_message(200, {'x-rucio-auth-token': 'sometoken'}, '')
+
+        start_time = datetime.utcnow()
+        with MockServer(FailTwiceWith502) as server:
+            creds = {'username': 'ddmlab', 'password': 'secret'}
+            del invocations[:]
+            client = BaseClient(rucio_host=server.base_url, auth_host=server.base_url, account='root', auth_type='userpass', creds=creds, **self.vo)
+            del invocations[:]
+            client._send_request(server.base_url)  # noqa
+        # The client did back-off multiple times before succeeding: 2 * 0.25s (authentication) + 2 * 0.25s (request) = 1s
+        assert datetime.now() - start_time > timedelta(seconds=0.9)
+
 
 class TestRucioClients(unittest.TestCase):
     """ To test Clients"""