Repository: rucio/rucio

Issue:
Add possibility to ask for Staging from DISK RSEs
Motivation
----------
Due to novel R&D in the area of QoS for storage systems, there are storages which are a hybrid between DISK & TAPE. However, in Rucio they need to be assigned one type. If the assignment is DISK all third party copy transfer requests will be done without asking for a STAGING/BRINGONLINE request, which will fail in certain conditions. The proposal of this issue is to add an RSE attribute `staging_required` (Name to be decided) which can be set also for DISK rses. For third-party-copy requests also staging requests are issued for these RSEs.



Discussion:

Resolving patch:
diff --git a/lib/rucio/core/replica.py b/lib/rucio/core/replica.py
--- a/lib/rucio/core/replica.py
+++ b/lib/rucio/core/replica.py
@@ -474,15 +474,15 @@ def list_bad_replicas(limit=10000, thread=None, total_threads=None, session=None
 
     :returns: a list of dictionary {'scope' scope, 'name': name, 'rse_id': rse_id, 'rse': rse}.
     """
-
+    schema_dot = '%s.' % DEFAULT_SCHEMA_NAME if DEFAULT_SCHEMA_NAME else ''
     if session.bind.dialect.name == 'oracle':
         # The filter(text...)) is needed otherwise, SQLA uses bind variables and the index is not used.
         query = session.query(models.RSEFileAssociation.scope,
                               models.RSEFileAssociation.name,
                               models.RSEFileAssociation.rse_id).\
             with_hint(models.RSEFileAssociation, "+ index(replicas REPLICAS_STATE_IDX)", 'oracle').\
-            filter(text("CASE WHEN (%s.replicas.state != 'A') THEN %s.replicas.rse_id END IS NOT NULL" % (DEFAULT_SCHEMA_NAME,
-                                                                                                          DEFAULT_SCHEMA_NAME))).\
+            filter(text("CASE WHEN (%sreplicas.state != 'A') THEN %sreplicas.rse_id END IS NOT NULL" % (schema_dot,
+                                                                                                        schema_dot))). \
             filter(models.RSEFileAssociation.state == ReplicaState.BAD)
     else:
         query = session.query(models.RSEFileAssociation.scope,
diff --git a/lib/rucio/core/transfer.py b/lib/rucio/core/transfer.py
--- a/lib/rucio/core/transfer.py
+++ b/lib/rucio/core/transfer.py
@@ -14,7 +14,7 @@
 #
 # Authors:
 # - Mario Lassnig <mario.lassnig@cern.ch>, 2013-2020
-# - Martin Barisits <martin.barisits@cern.ch>, 2017-2019
+# - Martin Barisits <martin.barisits@cern.ch>, 2017-2020
 # - Vincent Garonne <vgaronne@gmail.com>, 2017
 # - Igor Mandrichenko <rucio@fermicloud055.fnal.gov>, 2018
 # - Cedric Serfon <cedric.serfon@cern.ch>, 2018-2020
@@ -780,7 +780,7 @@ def get_transfer_requests_and_source_replicas(total_workers=0, worker_number=0,
 
                 # III - Extend the metadata dictionary with request attributes
                 overwrite, bring_online = True, None
-                if rses_info[source_rse_id]['rse_type'] == RSEType.TAPE or rses_info[source_rse_id]['rse_type'] == 'TAPE':
+                if rses_info[source_rse_id]['rse_type'] == RSEType.TAPE or rses_info[source_rse_id]['rse_type'] == 'TAPE' or rse_attrs[source_rse_id].get('staging_required', False):
                     bring_online = bring_online_local
                     transfer_src_type = "TAPE"
                     if not allow_tape_source:
@@ -973,7 +973,7 @@ def get_transfer_requests_and_source_replicas(total_workers=0, worker_number=0,
                 # TAPE should not mixed with Disk and should not use as first try
                 # If there is a source whose ranking is no less than the Tape ranking, Tape will not be used.
 
-                if rses_info[source_rse_id]['rse_type'] == RSEType.TAPE or rses_info[source_rse_id]['rse_type'] == 'TAPE':
+                if rses_info[source_rse_id]['rse_type'] == RSEType.TAPE or rses_info[source_rse_id]['rse_type'] == 'TAPE' or rse_attrs[source_rse_id].get('staging_required', False):
                     # current src_rse is Tape
                     if not allow_tape_source:
                         continue
diff --git a/lib/rucio/transfertool/globusLibrary.py b/lib/rucio/transfertool/globusLibrary.py
--- a/lib/rucio/transfertool/globusLibrary.py
+++ b/lib/rucio/transfertool/globusLibrary.py
@@ -14,14 +14,14 @@
 #
 # Authors:
 # - Matt Snyder <msnyder@rcf.rhic.bnl.gov>, 2019
-# - Martin Barisits <martin.barisits@cern.ch>, 2019
+# - Martin Barisits <martin.barisits@cern.ch>, 2019-2020
 
 import imp
 import logging
 import os
 import sys
 
-from rucio.common.config import config_get, get_config_dir
+from rucio.common.config import config_get, get_config_dirs
 from rucio.core.monitor import record_counter
 from datetime import datetime
 
@@ -52,7 +52,7 @@
 
 def load_config(cfg_file='globus-config.yml'):
     config = None
-    config_dir = get_config_dir()
+    config_dir = get_config_dirs()[0]
     if os.path.isfile(os.path.join(config_dir, cfg_file)):
         config = os.path.join(config_dir, cfg_file)
     else:


Test patch: