Repository: rucio/rucio

Issue:
Move conveyor transfers to third_party_copy_operation
Motivation
----------
At the moment read is used for source and write is used for destination, for both third_party_copy should be used.


Discussion:

Resolving patch:
diff --git a/lib/rucio/core/rse.py b/lib/rucio/core/rse.py
--- a/lib/rucio/core/rse.py
+++ b/lib/rucio/core/rse.py
@@ -17,7 +17,7 @@
 # - Ralph Vigne <ralph.vigne@cern.ch>, 2012-2015
 # - Mario Lassnig <mario.lassnig@cern.ch>, 2012-2017
 # - Martin Barisits <martin.barisits@cern.ch>, 2013-2018
-# - Cedric Serfon <cedric.serfon@cern.ch>, 2013-2016
+# - Cedric Serfon <cedric.serfon@cern.ch>, 2013-2018
 # - Thomas Beermann <thomas.beermann@cern.ch>, 2014-2017
 # - Wen Guan <wguan.icedew@gmail.com>, 2015-2016
 # - Brian Bockelman <bbockelm@cse.unl.edu>, 2018
@@ -280,10 +280,10 @@ def list_rses(filters={}, session=None):
 
         query = session.query(models.RSE).filter_by(deleted=False).order_by(models.RSE.rse)
         for row in query:
-            d = {}
+            dic = {}
             for column in row.__table__.columns:
-                d[column.name] = getattr(row, column.name)
-            rse_list.append(d)
+                dic[column.name] = getattr(row, column.name)
+            rse_list.append(dic)
 
     return rse_list
 
@@ -707,18 +707,18 @@ def add_protocol(rse, parameter, session=None):
         new_protocol = models.RSEProtocols()
         new_protocol.update(parameter)
         new_protocol.save(session=session)
-    except (IntegrityError, FlushError, OperationalError) as e:
-        if ('UNIQUE constraint failed' in e.args[0]) or ('conflicts with persistent instance' in e.args[0]) \
-           or match('.*IntegrityError.*ORA-00001: unique constraint.*RSE_PROTOCOLS_PK.*violated.*', e.args[0]) \
-           or match('.*IntegrityError.*1062.*Duplicate entry.*for key.*', e.args[0]) \
-           or match('.*IntegrityError.*duplicate key value violates unique constraint.*', e.args[0])\
-           or match('.*IntegrityError.*columns.*are not unique.*', e.args[0]):
+    except (IntegrityError, FlushError, OperationalError) as error:
+        if ('UNIQUE constraint failed' in error.args[0]) or ('conflicts with persistent instance' in error.args[0]) \
+           or match('.*IntegrityError.*ORA-00001: unique constraint.*RSE_PROTOCOLS_PK.*violated.*', error.args[0]) \
+           or match('.*IntegrityError.*1062.*Duplicate entry.*for key.*', error.args[0]) \
+           or match('.*IntegrityError.*duplicate key value violates unique constraint.*', error.args[0])\
+           or match('.*IntegrityError.*columns.*are not unique.*', error.args[0]):
             raise exception.Duplicate('Protocol \'%s\' on port %s already registered for  \'%s\' with hostname \'%s\'.' % (parameter['scheme'], parameter['port'], rse, parameter['hostname']))
-        elif 'may not be NULL' in e.args[0] \
-             or match('.*IntegrityError.*ORA-01400: cannot insert NULL into.*RSE_PROTOCOLS.*IMPL.*', e.args[0]) \
-             or match('.*OperationalError.*cannot be null.*', e.args[0]):
+        elif 'may not be NULL' in error.args[0] \
+             or match('.*IntegrityError.*ORA-01400: cannot insert NULL into.*RSE_PROTOCOLS.*IMPL.*', error.args[0]) \
+             or match('.*OperationalError.*cannot be null.*', error.args[0]):
             raise exception.InvalidObject('Missing values!')
-        raise e
+        raise error
     return new_protocol
 
 
@@ -853,10 +853,12 @@ def update_protocols(rse, scheme, data, hostname, port, session=None):
             for op in data['domains'][s]:
                 if op not in utils.rse_supported_protocol_operations():
                     raise exception.RSEOperationNotSupported('Operation \'%s\' not defined in schema.' % (op))
-                op_name = ''.join([op, '_', s])
+                op_name = op
+                if op != 'third_party_copy':
+                    op_name = ''.join([op, '_', s])
                 no = session.query(models.RSEProtocols).\
                     filter(sqlalchemy.and_(models.RSEProtocols.rse_id == rid,
-                                           getattr(models.RSEProtocols, op_name) > 0)).\
+                                           getattr(models.RSEProtocols, op_name) >= 0)).\
                     count()
                 if not 0 <= data['domains'][s][op] <= no:
                     raise exception.RSEProtocolPriorityError('The provided priority (%s)for operation \'%s\' in domain \'%s\' is not supported.' % (data['domains'][s][op], op, s))
@@ -886,7 +888,9 @@ def update_protocols(rse, scheme, data, hostname, port, session=None):
         # Preparing gaps if priority is updated
         for domain in utils.rse_supported_protocol_domains():
             for op in utils.rse_supported_protocol_operations():
-                op_name = ''.join([op, '_', domain])
+                op_name = op
+                if op != 'third_party_copy':
+                    op_name = ''.join([op, '_', domain])
                 if op_name in data:
                     prots = []
                     if (not getattr(up, op_name)) and data[op_name]:  # reactivate protocol e.g. from 0 to 1
@@ -921,12 +925,12 @@ def update_protocols(rse, scheme, data, hostname, port, session=None):
                         val += 1
 
         up.update(data, flush=True, session=session)
-    except (IntegrityError, OperationalError) as e:
-        if 'UNIQUE'.lower() in e.args[0].lower() or 'Duplicate' in e.args[0]:  # Covers SQLite, Oracle and MySQL error
+    except (IntegrityError, OperationalError) as error:
+        if 'UNIQUE'.lower() in error.args[0].lower() or 'Duplicate' in error.args[0]:  # Covers SQLite, Oracle and MySQL error
             raise exception.Duplicate('Protocol \'%s\' on port %s already registered for  \'%s\' with hostname \'%s\'.' % (scheme, port, rse, hostname))
-        elif 'may not be NULL' in e.args[0] or "cannot be null" in e.args[0]:
-            raise exception.InvalidObject('Missing values: %s' % e.args[0])
-        raise e
+        elif 'may not be NULL' in error.args[0] or "cannot be null" in error.args[0]:
+            raise exception.InvalidObject('Missing values: %s' % error.args[0])
+        raise error
     except DatabaseError as error:
         if match('.*DatabaseError.*ORA-01407: cannot update .*RSE_PROTOCOLS.*IMPL.*to NULL.*', error.args[0]):
             raise exception.InvalidObject('Invalid values !')
diff --git a/lib/rucio/core/transfer.py b/lib/rucio/core/transfer.py
--- a/lib/rucio/core/transfer.py
+++ b/lib/rucio/core/transfer.py
@@ -559,16 +559,15 @@ def get_transfer_requests_and_source_replicas(process=None, total_processes=None
                 allow_tape_source = True
 
                 # Find matching scheme between destination and source
-                # TODO: In the future, move to domain "third_party_copy"
                 try:
                     matching_scheme = rsemgr.find_matching_scheme(rse_settings_dest=rses_info[dest_rse_id],
                                                                   rse_settings_src=rses_info[source_rse_id],
-                                                                  operation_src='read',
-                                                                  operation_dest='write',
+                                                                  operation_src='third_party_copy',
+                                                                  operation_dest='third_party_copy',
                                                                   domain='wan',
                                                                   scheme=current_schemes)
                 except RSEProtocolNotSupported:
-                    logging.error('Operation "write" not supported by %s with schemes %s' % (rses_info[dest_rse_id]['rse'], current_schemes))
+                    logging.error('Operation "third_party_copy" not supported by %s with schemes %s' % (rses_info[dest_rse_id]['rse'], current_schemes))
                     if id in reqs_no_source:
                         reqs_no_source.remove(id)
                     if id not in reqs_scheme_mismatch:
@@ -578,9 +577,9 @@ def get_transfer_requests_and_source_replicas(process=None, total_processes=None
                 # Get destination protocol
                 if dest_rse_id not in protocols:
                     try:
-                        protocols[dest_rse_id] = rsemgr.create_protocol(rses_info[dest_rse_id], 'write', matching_scheme[0])
+                        protocols[dest_rse_id] = rsemgr.create_protocol(rses_info[dest_rse_id], 'third_party_copy', matching_scheme[0])
                     except RSEProtocolNotSupported:
-                        logging.error('Operation "write" not supported by %s with schemes %s' % (rses_info[dest_rse_id]['rse'], current_schemes))
+                        logging.error('Operation "third_party_copy" not supported by %s with schemes %s' % (rses_info[dest_rse_id]['rse'], current_schemes))
                         if id in reqs_no_source:
                             reqs_no_source.remove(id)
                         if id not in reqs_scheme_mismatch:
@@ -624,7 +623,7 @@ def get_transfer_requests_and_source_replicas(process=None, total_processes=None
                 source_rse_id_key = '%s_%s' % (source_rse_id, '_'.join([matching_scheme[0], matching_scheme[1]]))
                 if source_rse_id_key not in protocols:
                     try:
-                        protocols[source_rse_id_key] = rsemgr.create_protocol(rses_info[source_rse_id], 'read', matching_scheme[1])
+                        protocols[source_rse_id_key] = rsemgr.create_protocol(rses_info[source_rse_id], 'third_party_copy', matching_scheme[1])
                     except RSEProtocolNotSupported:
                         logging.error('Operation "read" not supported by %s with schemes %s' % (rses_info[source_rse_id]['rse'], matching_scheme[1]))
                         if id in reqs_no_source:
@@ -744,9 +743,9 @@ def get_transfer_requests_and_source_replicas(process=None, total_processes=None
                 source_rse_id_key = '%s_%s' % (source_rse_id, '_'.join(current_schemes))
                 if source_rse_id_key not in protocols:
                     try:
-                        protocols[source_rse_id_key] = rsemgr.create_protocol(rses_info[source_rse_id], 'read', current_schemes)
+                        protocols[source_rse_id_key] = rsemgr.create_protocol(rses_info[source_rse_id], 'third_party_copy', current_schemes)
                     except RSEProtocolNotSupported:
-                        logging.error('Operation "read" not supported by %s with schemes %s' % (rses_info[source_rse_id]['rse'], current_schemes))
+                        logging.error('Operation "third_party_copy" not supported by %s with schemes %s' % (rses_info[source_rse_id]['rse'], current_schemes))
                         continue
                 source_url = protocols[source_rse_id_key].lfns2pfns(lfns={'scope': scope, 'name': name, 'path': path}).values()[0]
 


Test patch: