Repository: rucio/rucio

Issue:
Probe to cancel FTS jobs for site in long downtime
Motivation
----------
If a site goes into a long downtime, but many transfers are already submitted to FTS, there will be a lot of failures.

Modification
------------
A probe collecting the downtimes in AGIS will cancel the FTS jobs if a site is supposed to be down for more than 24 hours.


Discussion:

Resolving patch:
diff --git a/lib/rucio/transfertool/fts3.py b/lib/rucio/transfertool/fts3.py
--- a/lib/rucio/transfertool/fts3.py
+++ b/lib/rucio/transfertool/fts3.py
@@ -13,6 +13,8 @@
 # - Eric Vaandering, <ewv@fnal.gov>, 2018
 # - Diego Ciangottini <diego.ciangottini@pg.infn.it>, 2018
 # - Cedric Serfon <cedric.serfon@cern.ch>, 2018
+#
+# PY3K COMPATIBLE
 
 from __future__ import absolute_import
 import datetime
@@ -24,7 +26,11 @@
 import logging
 import sys
 import time
-import urlparse
+import traceback
+try:
+    from urlparse import urlparse  # py2
+except ImportError:
+    from urllib.parse import urlparse  # py3
 import uuid
 
 import requests
@@ -393,7 +399,7 @@ def bulk_query(self, transfer_ids, timeout=None):
 
         jobs = None
 
-        if type(transfer_ids) is not list:
+        if not isinstance(transfer_ids, list):
             transfer_ids = [transfer_ids]
 
         responses = {}
@@ -427,6 +433,76 @@ def bulk_query(self, transfer_ids, timeout=None):
 
         return responses
 
+    def list_se_status(self):
+        """
+        Get the list of banned Storage Elements.
+
+        :returns: Detailed dictionnary of banned Storage Elements.
+        """
+
+        try:
+            result = requests.get('%s/ban/se' % self.external_host,
+                                  verify=self.verify,
+                                  cert=self.cert,
+                                  headers={'Content-Type': 'application/json'},
+                                  timeout=None)
+        except Exception as error:
+            raise Exception('Could not retrieve transfer information: %s', error)
+        if result and result.status_code == 200:
+            return result.json()
+        raise Exception('Could not retrieve transfer information: %s', result.content)
+
+    def set_se_status(self, storage_element, message, ban=True, timeout=None):
+        """
+        Ban a Storage Element. Used when a site is in downtime.
+        One can use a timeout in seconds. In that case the jobs will wait before being cancel.
+        If no timeout is specified, the jobs are canceled immediately
+
+        :param storage_element: The Storage Element that will be banned.
+        :param message: The reason of the ban.
+        :param ban: Boolean. If set to True, ban the SE, if set to False unban the SE.
+        :param timeout: if None, send to FTS status 'cancel' else 'waiting' + the corresponding timeout.
+
+        :returns: 0 in case of success, otherwise raise Exception
+        """
+
+        params_dict = {'storage': storage_element, 'message': message}
+        status = 'CANCEL'
+        if timeout:
+            params_dict['timeout'] = timeout
+            status = 'WAIT'
+        params_dict['status'] = status
+        params_str = json.dumps(params_dict)
+
+        result = None
+        if ban:
+            try:
+                result = requests.post('%s/ban/se' % self.external_host,
+                                       verify=self.verify,
+                                       cert=self.cert,
+                                       data=params_str,
+                                       headers={'Content-Type': 'application/json'},
+                                       timeout=None)
+            except Exception:
+                logging.warn('Could not ban %s on %s - %s', storage_element, self.external_host, str(traceback.format_exc()))
+            if result and result.status_code == 200:
+                return 0
+            raise Exception('Could not ban the storage %s , status code returned : %s', (storage_element, result.status_code if result else None))
+        else:
+
+            try:
+                result = requests.delete('%s/ban/se?storage=%s' % (self.external_host, storage_element),
+                                         verify=self.verify,
+                                         cert=self.cert,
+                                         data=params_str,
+                                         headers={'Content-Type': 'application/json'},
+                                         timeout=None)
+            except Exception:
+                logging.warn('Could not unban %s on %s - %s', storage_element, self.external_host, str(traceback.format_exc()))
+            if result and result.status_code == 204:
+                return 0
+            raise Exception('Could not unban the storage %s , status code returned : %s', (storage_element, result.status_code if result else None))
+
     # Private methods unique to the FTS3 Transfertool
 
     @staticmethod
@@ -617,7 +693,7 @@ def __format_new_response(self, fts_job_response, fts_files_response):
         return resps
 
     def __bulk_query_responses(self, jobs_response):
-        if type(jobs_response) is not list:
+        if not isinstance(jobs_response, list):
             jobs_response = [jobs_response]
 
         responses = {}
diff --git a/lib/rucio/transfertool/fts3_myproxy.py b/lib/rucio/transfertool/fts3_myproxy.py
--- a/lib/rucio/transfertool/fts3_myproxy.py
+++ b/lib/rucio/transfertool/fts3_myproxy.py
@@ -34,9 +34,9 @@
 from dogpile.cache import make_region
 from dogpile.cache.api import NoValue
 
-from fts3.rest.client.exceptions import BadEndpoint, ClientError, ServerError
-import fts3.rest.client.easy as fts
-from myproxy.client import MyProxyClient, MyProxyClientGetError, MyProxyClientRetrieveError
+from fts3.rest.client.exceptions import BadEndpoint, ClientError, ServerError  # pylint: disable=no-name-in-module,import-error
+import fts3.rest.client.easy as fts  # pylint: disable=no-name-in-module,import-error
+from myproxy.client import MyProxyClient, MyProxyClientGetError, MyProxyClientRetrieveError  # pylint: disable=no-name-in-module,import-error
 
 from rucio.common.config import config_get, config_get_bool
 from rucio.core.monitor import record_counter
@@ -128,7 +128,7 @@ def get_dn_from_scope(self, scope, ca_path='/etc/grid-security/certificates/', s
             logging.error("SiteDB request exception")
             raise
 
-        if resp.status_code == requests.codes.ok:
+        if resp.status_code == requests.codes.ok:  # pylint: disable=no-member
             resp_json = resp.json()
         else:
             logging.error("Bad SiteDB request status code")
@@ -195,8 +195,7 @@ def get_user_proxy(self, myproxy_server, userDN, force_remote=False):
             cert = myproxy_client.logon(key,
                                         None,
                                         sslCertFile=cert,
-                                        sslKeyFile=ckey
-                                        )
+                                        sslKeyFile=ckey)
         except MyProxyClientGetError:
             logging.error("MyProxy client exception during GET proxy")
             raise


Test patch: