Repository: rucio/rucio

Issue:
Fix source scheme in case of transfers from GCS
Motivation
----------
Fix source scheme in case of transfers from GCS



add "thread" support to lock and replica retrieval functions
Motivation
----------
needed to address scalability problems of add_rule


Modification
------------
/lib/rucio/core/lock.py
add total_threads/thread_id support to get_files_and_replica_locks_of_dataset
/lib/rucio/core/replica.py
add total_threads/thread_id support to get_and_lock_file_replicas_for_dataset
add total_threads/thread_id support to get_source_replicas_for_dataset
 


Rucio client package imports rucio.common.schema which is not part of distribution
Motivation
----------
From Isabel:
```
cmd: rucio upload --scope user.itrigger --rse TRIUMF-LCG2_SCRATCHDISK /tmp/itrigger/itrigger/.alrb/testing/tester.NKHUnK/rucio/bash/rucioUpload
2020-04-06 18:30:38,129
INFO Preparing upload for file myFile.d314ec0d-6684-4e3f-8501-47467586e335.log
2020-04-06 18:30:38,293
INFO Successfully added replica in Rucio catalogue at TRIUMF-LCG2_SCRATCHDISK
2020-04-06 18:30:38,842
INFO Successfully added replication rule at TRIUMF-LCG2_SCRATCHDISK
2020-04-06 18:30:39,962
ERROR No module named schema
2020-04-06 18:30:39,963
ERROR 
Rucio exited with an unexpected/unknown error.
Please rerun the last command with the "-v" option to gather more information.
If you're sure there is a problem with Rucio itself, please followup at: https://github.com/rucio/rucio/issues
rucio upload dataset (bash)                                         [FAILED]
```
For some reason the schema module seems to be imported, which is not part of the client distribution.

Modification
------------
We can add it to the distribution, but for me it is not clear why this is even needed, so we should understand this first.



Discussion:


line where we import the schema in client mode:
https://github.com/rucio/rucio/blob/next/lib/rucio/rse/protocols/protocol.py#L230
Ok, I followed this up and I think in this case we have to include common.schema as well to the clients package, as it is needed for the VO policy packages. I will prepare a PR.

Resolving patch:
diff --git a/lib/rucio/core/lock.py b/lib/rucio/core/lock.py
--- a/lib/rucio/core/lock.py
+++ b/lib/rucio/core/lock.py
@@ -11,6 +11,7 @@
 # - Vincent Garonne, <vincent.garonne@cern.ch>, 2014-2018
 # - Cedric Serfon, <cedric.serfon@cern.ch>, 2014-2018
 # - Thomas Beermann, <thomas.beermann@cern.ch>, 2014-2018
+# - Luc Goossens <luc.goossens@cern.ch>, 2020
 #
 # PY3K COMPATIBLE
 
@@ -29,7 +30,7 @@
 from rucio.common.exception import RSENotFound
 from rucio.core.lifetime_exception import define_eol
 from rucio.core.rse import get_rse_name, get_rse_id
-from rucio.db.sqla import models
+from rucio.db.sqla import models, filter_thread_work
 from rucio.db.sqla.constants import LockState, RuleState, RuleGrouping, DIDType, RuleNotification
 from rucio.db.sqla.session import read_session, transactional_session, stream_session
 
@@ -182,7 +183,9 @@ def get_replica_locks_for_rule_id_per_rse(rule_id, session=None):
 
 
 @read_session
-def get_files_and_replica_locks_of_dataset(scope, name, nowait=False, restrict_rses=None, only_stuck=False, session=None):
+def get_files_and_replica_locks_of_dataset(scope, name, nowait=False, restrict_rses=None, only_stuck=False,
+                                           total_threads=None, thread_id=None,
+                                           session=None):
     """
     Get all the files of a dataset and, if existing, all locks of the file.
 
@@ -191,6 +194,8 @@ def get_files_and_replica_locks_of_dataset(scope, name, nowait=False, restrict_r
     :param nowait:         Nowait parameter for the FOR UPDATE statement
     :param restrict_rses:  Possible RSE_ids to filter on.
     :param only_stuck:     If true, only get STUCK locks.
+    :param total_threads:  Total threads
+    :param thread_id:      This thread
     :param session:        The db session.
     :return:               Dictionary with keys: (scope, name)
                            and as value: [LockObject]
@@ -206,6 +211,10 @@ def get_files_and_replica_locks_of_dataset(scope, name, nowait=False, restrict_r
             filter(models.DataIdentifierAssociation.scope == scope,
                    models.DataIdentifierAssociation.name == name)
 
+        if total_threads and total_threads > 1:
+            content_query = filter_thread_work(session=session, query=content_query, total_threads=total_threads,
+                                               thread_id=thread_id, hash_variable='child_name')
+
         for child_scope, child_name in content_query.yield_per(1000):
             locks[(child_scope, child_name)] = []
 
@@ -263,6 +272,10 @@ def get_files_and_replica_locks_of_dataset(scope, name, nowait=False, restrict_r
     if only_stuck:
         query = query.filter(models.ReplicaLock.state == LockState.STUCK)
 
+    if total_threads and total_threads > 1:
+        query = filter_thread_work(session=session, query=query, total_threads=total_threads,
+                                   thread_id=thread_id, hash_variable='child_name')
+
     query = query.with_for_update(nowait=nowait, of=models.ReplicaLock.state)
 
     for child_scope, child_name, lock in query:
diff --git a/lib/rucio/core/replica.py b/lib/rucio/core/replica.py
--- a/lib/rucio/core/replica.py
+++ b/lib/rucio/core/replica.py
@@ -1961,6 +1961,7 @@ def get_source_replicas(scope, name, source_rses=None, session=None):
 
 @transactional_session
 def get_and_lock_file_replicas_for_dataset(scope, name, nowait=False, restrict_rses=None,
+                                           total_threads=None, thread_id=None,
                                            session=None):
     """
     Get file replicas for all files of a dataset.
@@ -1969,6 +1970,8 @@ def get_and_lock_file_replicas_for_dataset(scope, name, nowait=False, restrict_r
     :param name:           The name of the dataset.
     :param nowait:         Nowait parameter for the FOR UPDATE statement
     :param restrict_rses:  Possible RSE_ids to filter on.
+    :param total_threads:  Total threads
+    :param thread_id:      This thread
     :param session:        The db session in use.
     :returns:              (files in dataset, replicas in dataset)
     """
@@ -1986,6 +1989,10 @@ def get_and_lock_file_replicas_for_dataset(scope, name, nowait=False, restrict_r
             filter(models.DataIdentifierAssociation.scope == scope,
                    models.DataIdentifierAssociation.name == name)
 
+        if total_threads and total_threads > 1:
+            content_query = filter_thread_work(session=session, query=content_query, total_threads=total_threads,
+                                               thread_id=thread_id, hash_variable='child_name')
+
         for child_scope, child_name, bytes, md5, adler32 in content_query.yield_per(1000):
             files[(child_scope, child_name)] = {'scope': child_scope,
                                                 'name': child_name,
@@ -2072,6 +2079,10 @@ def get_and_lock_file_replicas_for_dataset(scope, name, nowait=False, restrict_r
                                    .filter(models.DataIdentifierAssociation.scope == scope,
                                            models.DataIdentifierAssociation.name == name)
 
+    if total_threads and total_threads > 1:
+        query = filter_thread_work(session=session, query=query, total_threads=total_threads,
+                                   thread_id=thread_id, hash_variable='child_name')
+
     query = query.with_for_update(nowait=nowait, of=models.RSEFileAssociation.lock_cnt)
 
     for child_scope, child_name, bytes, md5, adler32, replica in query.yield_per(1000):
@@ -2094,13 +2105,17 @@ def get_and_lock_file_replicas_for_dataset(scope, name, nowait=False, restrict_r
 
 
 @transactional_session
-def get_source_replicas_for_dataset(scope, name, source_rses=None, session=None):
+def get_source_replicas_for_dataset(scope, name, source_rses=None,
+                                    total_threads=None, thread_id=None,
+                                    session=None):
     """
     Get file replicas for all files of a dataset.
 
     :param scope:          The scope of the dataset.
     :param name:           The name of the dataset.
     :param source_rses:    Possible source RSE_ids to filter on.
+    :param total_threads:  Total threads
+    :param thread_id:      This thread
     :param session:        The db session in use.
     :returns:              (files in dataset, replicas in dataset)
     """
@@ -2132,6 +2147,10 @@ def get_source_replicas_for_dataset(scope, name, source_rses=None, session=None)
                                .filter(models.DataIdentifierAssociation.scope == scope,
                                        models.DataIdentifierAssociation.name == name)
 
+    if total_threads and total_threads > 1:
+        query = filter_thread_work(session=session, query=query, total_threads=total_threads,
+                                   thread_id=thread_id, hash_variable='child_name')
+
     replicas = {}
 
     for child_scope, child_name, rse_id in query:
diff --git a/lib/rucio/core/transfer.py b/lib/rucio/core/transfer.py
--- a/lib/rucio/core/transfer.py
+++ b/lib/rucio/core/transfer.py
@@ -800,9 +800,16 @@ def get_transfer_requests_and_source_replicas(total_workers=0, worker_number=0,
                     if source_protocol in ['davs', 'https']:
                         source_url += '?copy_mode=%s' % WEBDAV_TRANSFER_MODE
 
+                source_sign_url = rse_attrs[source_rse_id].get('sign_url', None)
+                if source_sign_url == 'gcs':
+                    source_url = re.sub('davs', 'gclouds', source_url)
+                    source_url = re.sub('https', 'gclouds', source_url)
+
                 # IV - get external_host + strict_copy
                 strict_copy = rse_attrs[dest_rse_id].get('strict_copy', False)
                 fts_hosts = rse_attrs[dest_rse_id].get('fts', None)
+                if source_sign_url == 'gcs':
+                    fts_hosts = rse_attrs[source_rse_id].get('fts', None)
                 source_globus_endpoint_id = rse_attrs[source_rse_id].get('globus_endpoint_id', None)
                 dest_globus_endpoint_id = rse_attrs[dest_rse_id].get('globus_endpoint_id', None)
 
@@ -946,6 +953,11 @@ def get_transfer_requests_and_source_replicas(total_workers=0, worker_number=0,
                     if source_protocol in ['davs', 'https']:
                         source_url += '?copy_mode=%s' % WEBDAV_TRANSFER_MODE
 
+                source_sign_url = rse_attrs[source_rse_id].get('sign_url', None)
+                if source_sign_url == 'gcs':
+                    source_url = re.sub('davs', 'gclouds', source_url)
+                    source_url = re.sub('https', 'gclouds', source_url)
+
                 # III - The transfer queued previously is a multihop, but this one is direct.
                 # Reset the sources, remove the multihop flag
                 if transfers[req_id].get('multihop', False):


Test patch: