Repository: rucio/rucio
Issue: Conveyor breaks if no scheme is specified

Motivation
----------

When no scheme is specified in the configuration, the conveyor fails to find a common scheme for the source and destination files, here `gsiftp`:

```
2019-03-27 08:51:47,386	3214	CRITICAL	Exception happened when trying to get transfer for request 5d9c6d99ebf04465ad3bef4d8078219c: Traceback (most recent call last):
  File "/usr/local/lib/python2.7/dist-packages/rucio/core/transfer.py", line 624, in get_transfer_requests_and_source_replicas
    'schemes': __add_compatible_schemes(schemes=[matching_scheme[0]], allowed_schemes=current_schemes),
  File "/usr/local/lib/python2.7/dist-packages/rucio/core/transfer.py", line 923, in __add_compatible_schemes
    if scheme in allowed_schemes:
TypeError: argument of type 'NoneType' is not iterable
```

The respective rules stay in the STUCK state when asked for reevaluation:

```
2019-03-27 08:49:23,677	3213	INFO	rule_repairer[0/0]: Repairing rule 32d6775ead104b2c8be50cc946f01782
2019-03-27 08:49:23,749	3213	INFO	Rule 32d6775ead104b2c8be50cc946f01782 [0/0/1] state=STUCK
```


Changed files:
lib/rucio/rse/__init__.py, lib/rucio/core/transfer.py, lib/rucio/web/rest/webpy/v1/replica.py, lib/rucio/rse/protocols/__init__.py, lib/rucio/common/constants.py

diff --git a/lib/rucio/common/constants.py b/lib/rucio/common/constants.py
index 6d7635882..452c8d03b 100644
--- a/lib/rucio/common/constants.py
+++ b/lib/rucio/common/constants.py
@@ -6,7 +6,7 @@
 #
 # Authors:
 # - Vincent Garonne, <vincent.garonne@cern.ch>, 2013
-# - Cedric Serfon, <cedric.serfon@cern.ch>, 2015-2018
+# - Cedric Serfon, <cedric.serfon@cern.ch>, 2015-2019
 # - Mario Lassnig, <mario.lassnig@cern.ch>, 2018
 #
 # PY3K COMPATIBLE
@@ -31,3 +31,5 @@ SCHEME_MAP = {'srm': ['srm', 'gsiftp'],
               'davs': ['https', 'davs', 's3'],
               'root': ['root'],
               's3': ['https', 'davs', 's3']}
+
+SUPPORTED_PROTOCOLS = ['gsiftp', 'srm', 'root', 'davs', 'http', 'https', 'file', 's3', 's3+rucio', 's3+https', 'storm']
diff --git a/lib/rucio/core/transfer.py b/lib/rucio/core/transfer.py
index 8b5179897..6c1f140d5 100644
--- a/lib/rucio/core/transfer.py
+++ b/lib/rucio/core/transfer.py
@@ -32,6 +32,7 @@ from rucio.common import constants
 from rucio.common.exception import RucioException, UnsupportedOperation, InvalidRSEExpression, RSEProtocolNotSupported, RequestNotFound
 from rucio.common.rse_attributes import get_rse_attributes
 from rucio.common.utils import construct_surl
+from rucio.common.constants import SUPPORTED_PROTOCOLS
 from rucio.core import did, message as message_core, request as request_core
 from rucio.core.monitor import record_counter, record_timer
 from rucio.core.rse import get_rse_name, list_rses
@@ -433,7 +434,9 @@ def get_transfer_requests_and_source_replicas(total_workers=0, worker_number=0,
             if rses and dest_rse_id not in rses:
                 continue
 
-            current_schemes = schemes
+            current_schemes = SUPPORTED_PROTOCOLS
+            if schemes:
+                current_schemes = schemes
             if previous_attempt_id and failover_schemes:
                 current_schemes = failover_schemes
 
diff --git a/lib/rucio/rse/__init__.py b/lib/rucio/rse/__init__.py
index 2716cc068..6231f3184 100644
--- a/lib/rucio/rse/__init__.py
+++ b/lib/rucio/rse/__init__.py
@@ -1,16 +1,24 @@
-'''
- Copyright European Organization for Nuclear Research (CERN)
-
- Licensed under the Apache License, Version 2.0 (the "License");
- You may not use this file except in compliance with the License.
- You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
-
- Authors:
- - Ralph Vigne, <ralph.vigne@cern.ch>, 2013 - 2014
- - Vincent Garonne, <vincent.garonne@cern.ch>, 2013-2017
- - Cedric Serfon, <cedric.serfon@cern.ch>, 2017
- - James Perry, <j.perry@epcc.ed.ac.uk>, 2019
-'''
+# Copyright 2013-2019 CERN for the benefit of the ATLAS collaboration.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+# Authors:
+# - Ralph Vigne, <ralph.vigne@cern.ch>, 2013 - 2014
+# - Vincent Garonne, <vincent.garonne@cern.ch>, 2013-2017
+# - Cedric Serfon, <cedric.serfon@cern.ch>, 2017-2019
+# - James Perry, <j.perry@epcc.ed.ac.uk>, 2019
+#
+# PY3K COMPATIBLE
 
 from dogpile.cache import make_region
 
diff --git a/lib/rucio/rse/protocols/__init__.py b/lib/rucio/rse/protocols/__init__.py
index 1e444862d..83771847d 100644
--- a/lib/rucio/rse/protocols/__init__.py
+++ b/lib/rucio/rse/protocols/__init__.py
@@ -1,10 +1,15 @@
-# Copyright European Organization for Nuclear Research (CERN)
+# Copyright 2013-2019 CERN for the benefit of the ATLAS collaboration.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
-# You may not use this file except in compliance with the License.
-# You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0
-
-# These are all protocols which are considered in good health
-# based on generally available support libraries, native dependencies,
-# and implementations in Rucio
-supported_protocols = ['gsiftp', 'srm', 'root', 'davs', 'http', 'https', 'file', 's3', 's3+rucio', 's3+https', 'storm']
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+# PY3K COMPATIBLE
diff --git a/lib/rucio/web/rest/webpy/v1/replica.py b/lib/rucio/web/rest/webpy/v1/replica.py
index e4f3cebd5..45b94422b 100755
--- a/lib/rucio/web/rest/webpy/v1/replica.py
+++ b/lib/rucio/web/rest/webpy/v1/replica.py
@@ -56,7 +56,7 @@ from rucio.common.exception import (AccessDenied, DataIdentifierAlreadyExists, I
 from rucio.common.replica_sorter import sort_random, sort_geoip, sort_closeness, sort_dynamic, sort_ranking
 from rucio.common.schema import SCOPE_NAME_REGEXP
 from rucio.common.utils import generate_http_error, parse_response, APIEncoder, render_json_list
-from rucio.rse.protocols import supported_protocols
+from rucio.common.constants import SUPPORTED_PROTOCOLS
 from rucio.web.rest.common import rucio_loadhook, rucio_unloadhook, RucioController, check_accept_header_wrapper
 
 URLS = ('/list/?$', 'ListReplicas',
@@ -111,7 +111,7 @@ class Replicas(RucioController):
 
         # Resolve all reasonable protocols when doing metalink for maximum access possibilities
         if metalink and schemes is None:
-            schemes = supported_protocols
+            schemes = SUPPORTED_PROTOCOLS
 
         try:
 
@@ -368,7 +368,7 @@ class ListReplicas(RucioController):
 
         # Resolve all reasonable protocols when doing metalink for maximum access possibilities
         if metalink and schemes is None:
-            schemes = supported_protocols
+            schemes = SUPPORTED_PROTOCOLS
 
         try: