Repository: rucio/rucio

Issue:
Failure with multihop in case an intermediate request already exists
Motivation
----------
```
2019-11-07 13:42:04,200 6560    CRITICAL        Thread [1/1] :  Traceback (most recent call last):
  File "/usr/lib/python2.7/site-packages/rucio/daemons/conveyor/submitter.py", line 165, in submitter
    retry_other_fts=retry_other_fts)
  File "/usr/lib/python2.7/site-packages/rucio/daemons/conveyor/submitter.py", line 317, in __get_transfers
    failover_schemes=failover_schemes)
  File "/usr/lib/python2.7/site-packages/rucio/db/sqla/session.py", line 365, in new_funct
    result = function(*args, **kwargs)
  File "/usr/lib/python2.7/site-packages/rucio/core/transfer.py", line 1015, in get_transfer_requests_and_source_replicas
    new_req_id = new_req[0]['id']
IndexError: list index out of range
```







Discussion:

Resolving patch:
diff --git a/lib/rucio/core/transfer.py b/lib/rucio/core/transfer.py
--- a/lib/rucio/core/transfer.py
+++ b/lib/rucio/core/transfer.py
@@ -43,7 +43,7 @@
 from rucio.core import did, message as message_core, request as request_core
 from rucio.core.monitor import record_counter, record_timer
 from rucio.core.replica import add_replicas
-from rucio.core.request import queue_requests, set_request_state
+from rucio.core.request import queue_requests, set_requests_state
 from rucio.core.rse import get_rse_name, list_rses
 from rucio.core.rse_expression_parser import parse_expression
 from rucio.db.sqla import models
@@ -453,7 +453,6 @@ def get_hops(source_rse_id, dest_rse_id, include_multihop=True, session=None):
     :raises:                   NoDistance
     """
 
-    logging.debug('Starting get_hops')
     # TODO: Might be problematic to always load the distance_graph, since it might be expensive
     # TODO: Have an rse_expression to specify the eligible hops
 
@@ -621,7 +620,7 @@ def get_transfer_requests_and_source_replicas(total_workers=0, worker_number=0,
         if source_rse_id in unavailable_read_rse_ids:
             continue
         if dest_rse_id in unavailable_write_rse_ids:
-            logging.warning('RSE %s is blacklisted for write. Will skip the submission of new jobs' % (dest_rse_name))
+            logging.warning('RSE %s is blacklisted for write. Will skip the submission of new jobs', dest_rse_name)
             continue
 
         # Call the get_hops function to create a list of RSEs used for the transfer
@@ -634,14 +633,14 @@ def get_transfer_requests_and_source_replicas(total_workers=0, worker_number=0,
                 multihop = True
                 multi_hop_dict[req_id] = (list_hops, dict_attributes, retry_count)
         except NoDistance:
-            logging.warning("Request %s: no link from %s to %s" % (req_id, source_rse_name, dest_rse_name))
+            logging.warning("Request %s: no link from %s to %s", req_id, source_rse_name, dest_rse_name)
             if req_id in reqs_scheme_mismatch:
                 reqs_scheme_mismatch.remove(req_id)
             if req_id not in reqs_no_source:
                 reqs_no_source.append(req_id)
             continue
         except RSEProtocolNotSupported:
-            logging.warning("Request %s: no matching protocol between %s and %s" % (req_id, source_rse_name, dest_rse_name))
+            logging.warning("Request %s: no matching protocol between %s and %s", req_id, source_rse_name, dest_rse_name)
             if req_id in reqs_no_source:
                 reqs_no_source.remove(req_id)
             if req_id not in reqs_scheme_mismatch:
@@ -675,7 +674,7 @@ def get_transfer_requests_and_source_replicas(total_workers=0, worker_number=0,
                 try:
                     parsed_rses = parse_expression(source_replica_expression, session=session)
                 except InvalidRSEExpression as error:
-                    logging.error("Invalid RSE exception %s: %s" % (source_replica_expression, error))
+                    logging.error("Invalid RSE exception %s: %s", source_replica_expression, str(error))
                     continue
                 else:
                     allowed_rses = [x['id'] for x in parsed_rses]
@@ -754,10 +753,10 @@ def get_transfer_requests_and_source_replicas(total_workers=0, worker_number=0,
                 dest_globus_endpoint_id = rse_attrs[dest_rse_id].get('globus_endpoint_id', None)
 
                 if TRANSFER_TOOL == 'fts3' and not fts_hosts:
-                    logging.error('Destination RSE %s FTS attribute not defined - SKIP REQUEST %s' % (dest_rse_name, req_id))
+                    logging.error('Destination RSE %s FTS attribute not defined - SKIP REQUEST %s', dest_rse_name, req_id)
                     continue
                 if TRANSFER_TOOL == 'globus' and (not dest_globus_endpoint_id or not source_globus_endpoint_id):
-                    logging.error('Destination RSE %s Globus endpoint attributes not defined - SKIP REQUEST %s' % (dest_rse_name, req_id))
+                    logging.error('Destination RSE %s Globus endpoint attributes not defined - SKIP REQUEST %s', dest_rse_name, req_id)
                     continue
                 if retry_count is None:
                     retry_count = 0
@@ -957,6 +956,7 @@ def get_transfer_requests_and_source_replicas(total_workers=0, worker_number=0,
             scope = transfers[req_id]['file_metadata']['scope']
             name = transfers[req_id]['file_metadata']['name']
             list_multihop, dict_attributes, retry_count = multi_hop_dict[req_id]
+            parent_requests = []
 
             for hop in list_multihop:
                 # hop = {'source_rse_id': source_rse_id, 'source_scheme': 'srm', 'source_scheme_priority': N, 'dest_rse_id': dest_rse_id, 'dest_scheme': 'srm', 'dest_scheme_priority': N}
@@ -1010,11 +1010,20 @@ def get_transfer_requests_and_source_replicas(total_workers=0, worker_number=0,
                                                         'retry_count': retry_count,
                                                         'account': InternalAccount('root'),
                                                         'requested_at': datetime.datetime.now()}], session=session)
-                    # TODO : This assume that a new request is created, but if a request already exists
-                    # new_req will be an empty list.
+                    # If a request already exists, new_req will be an empty list.
+                    if not new_req:
+                        # Need to fail all the intermediate requests + the initial one and exit the multihop loop
+                        logging.warning('Multihop : A request already exists for the transfer between %s and %s. Will cancel all the parent requests', source_rse_name, dest_rse_name)
+                        parent_requests.append(req_id)
+                        set_requests_state(request_ids=parent_requests, new_state=RequestState.FAILED, session=session)
+                        # Remove from the transfer dictionary all the requests
+                        for cur_req_id in parent_requests:
+                            transfers.pop(cur_req_id, None)
+                        break
                     new_req_id = new_req[0]['id']
-                    set_request_state(request_id=new_req_id, new_state=RequestState.QUEUED, session=session)
-                    logging.debug('New request created for the the transfer between %s and %s : %s', source_rse_name, dest_rse_name, new_req_id)
+                    parent_requests.append(new_req_id)
+                    set_requests_state(request_ids=[new_req_id, ], new_state=RequestState.QUEUED, session=session)
+                    logging.debug('New request created for the transfer between %s and %s : %s', source_rse_name, dest_rse_name, new_req_id)
 
                     # I - Here we will compute the destination URL
                     # I.1 - Get destination protocol


Test patch: