Repository: rucio/rucio

Issue:
Format error with `rucio download`
Motivation
----------

```
# rucio --version 
rucio 1.19.4
# rucio download   "238357;238356;238359:manda4_cp1_3.00u_SW20140322.part1.tar"
2019-03-21 13:58:45,998    INFO    Processing 1 item(s) for input
2019-03-21 13:58:45,998    INFO    Getting sources of DIDs
2019-03-21 13:58:46,434    ERROR    %d format: a number is required, not NoneType
```







Discussion:
```
(.venv) [13:49][twegner@twegner-dev rucio]$ rucio download "238357;238356;238359:manda4_cp1_3.00u_SW20140322.part1.tar"
2019-03-25 13:50:19,406	INFO	Processing 1 item(s) for input
2019-03-25 13:50:19,406	INFO	Getting sources of DIDs
2019-03-25 13:50:20,992	INFO	Using main thread to download 0 file(s)
2019-03-25 13:50:20,993	ERROR	None of the requested files have been downloaded.
```
could you please give debug output?
```
# rucio -v  download  "238357;238356;238359:manda4_cp1_3.00u_SW20140322.part1.tar"  --rse EISCAT 
2019-03-25 12:56:10,672	INFO	Processing 1 item(s) for input
2019-03-25 12:56:10,673	DEBUG	num_unmerged_items=1; num_dids=1; num_merged_items=1
2019-03-25 12:56:10,673	INFO	Getting sources of DIDs
2019-03-25 12:56:10,673	DEBUG	schemes: ['davs', 'gsiftp', 'https', 'root', 'srm', 'file']
2019-03-25 12:56:10,673	DEBUG	rse_expression: (EISCAT)\istape=true
2019-03-25 12:56:10,673	DEBUG	num DIDs for list_replicas call: 1
2019-03-25 12:56:10,942	DEBUG	num resolved files: 1
2019-03-25 12:56:10,949	DEBUG	"unzip -v" returned with exitcode 127
2019-03-25 12:56:10,954	DEBUG	"tar --version" returned with exitcode 0
2019-03-25 12:56:10,954	DEBUG	num list_replicas calls: 1
2019-03-25 12:56:10,954	DEBUG	Queueing file: 238357;238356;238359:manda4_cp1_3.00u_SW20140322.part1.tar
2019-03-25 12:56:10,955	DEBUG	real parents: set([])
2019-03-25 12:56:10,955	DEBUG	options: {'238357;238356;238359:manda4_cp1_3.00u_SW20140322.part1.tar': {'ignore_checksum': False, 'transfer_timeout': 3600, 'destinations': set([('.', False)])}}
2019-03-25 12:56:10,955	DEBUG	Prepared sources: num_sources=0/0; num_non_cea_sources=0; num_cea_ids=0
2019-03-25 12:56:10,956	ERROR	%d format: a number is required, not NoneType
2019-03-25 12:56:10,956	DEBUG	This means the parameter you passed has a wrong type.
Completed in 0.2852 sec.
```

Resolving patch:
diff --git a/lib/rucio/client/downloadclient.py b/lib/rucio/client/downloadclient.py
--- a/lib/rucio/client/downloadclient.py
+++ b/lib/rucio/client/downloadclient.py
@@ -17,7 +17,7 @@
 # - Vincent Garonne <vgaronne@gmail.com>, 2018
 # - Joaquin Bogado <jbogado@linti.unlp.edu.ar>, 2018
 # - Nicolo Magini <nicolo.magini@cern.ch>, 2018-2019
-# - Tobias Wegner <tobias.wegner@cern.ch>, 2018
+# - Tobias Wegner <tobias.wegner@cern.ch>, 2018-2019
 # - Hannes Hansen <hannes.jakob.hansen@cern.ch>, 2018-2019
 #
 # PY3K COMPATIBLE
@@ -1142,11 +1142,9 @@ def _get_sources(self, merged_items, resolve_archives=True):
         merged_items_with_sources = []
         for item in merged_items:
             # since we're using metalink we need to explicitly give all schemes
-            force_scheme = item.get('force_scheme')
-            if force_scheme:
-                schemes = force_scheme if isinstance(force_scheme, list) else [force_scheme]
-            else:
-                schemes = ['davs', 'gsiftp', 'https', 'root', 'srm', 'file']
+            schemes = item.get('force_scheme')
+            if schemes:
+                schemes = schemes if isinstance(schemes, list) else [schemes]
             logger.debug('schemes: %s' % schemes)
 
             # extend RSE expression to exclude tape RSEs for non-admin accounts
@@ -1313,9 +1311,12 @@ def _prepare_items_for_download(self, did_to_options, merged_items_with_sources,
 
                     file_item['sources'] = sources
 
+                    # if there are no cea sources we are done for this item
+                    if min_cea_priority is None:
+                        continue
                     # decide if file item belongs to the pure or mixed map
                     # if no non-archive src exists or the highest prio src is an archive src we put it in the pure map
-                    if num_non_cea_sources == 0 or min_cea_priority == 1:
+                    elif num_non_cea_sources == 0 or min_cea_priority == 1:
                         logger.debug('Adding fiid to cea pure map: '
                                      'num_non_cea_sources=%d; min_cea_priority=%d; num_cea_sources=%d'
                                      % (num_non_cea_sources, min_cea_priority, len(cea_ids)))
diff --git a/lib/rucio/rse/protocols/__init__.py b/lib/rucio/rse/protocols/__init__.py
--- a/lib/rucio/rse/protocols/__init__.py
+++ b/lib/rucio/rse/protocols/__init__.py
@@ -7,4 +7,4 @@
 # These are all protocols which are considered in good health
 # based on generally available support libraries, native dependencies,
 # and implementations in Rucio
-supported_protocols = ['gsiftp', 'srm', 'root', 'davs', 'https', 'file', 's3', 's3+rucio', 's3+https']
+supported_protocols = ['gsiftp', 'srm', 'root', 'davs', 'http', 'https', 'file', 's3', 's3+rucio', 's3+https']


Test patch:
diff --git a/lib/rucio/tests/test_download.py b/lib/rucio/tests/test_download.py
--- a/lib/rucio/tests/test_download.py
+++ b/lib/rucio/tests/test_download.py
@@ -17,37 +17,57 @@
 #
 # PY3K COMPATIBLE
 
-from nose.tools import assert_true
+import logging
 
+import nose.tools
+import os.path
+
+from rucio.client.client import Client
 from rucio.client.downloadclient import DownloadClient
+from rucio.client.uploadclient import UploadClient
 from rucio.common.utils import generate_uuid
-from rucio.tests.common import execute, file_generator
+from rucio.tests.common import file_generator
 
 
 class TestDownloadClient(object):
 
+    def setup(self):
+        logger = logging.getLogger('dlul_client')
+        logger.addHandler(logging.StreamHandler())
+        logger.setLevel(logging.DEBUG)
+        self.client = Client()
+        self.upload_client = UploadClient(_client=self.client, logger=logger)
+        self.download_client = DownloadClient(client=self.client, logger=logger)
+
+    def create_and_upload_tmp_file(self, rse, scope='mock'):
+        file_path = file_generator()
+        item = {'path': file_path,
+                'rse': rse,
+                'did_scope': scope,
+                'did_name': os.path.basename(file_path),
+                'guid': generate_uuid()}
+        nose.tools.assert_equal(self.upload_client.upload([item]), 0)
+        return item
+
     def test_download_item(self):
         """ DOWNLOAD (CLIENT): download DIDs. """
-        download_client = DownloadClient()
-        tmp_file1 = file_generator()
-        scope = 'mock'
-        name = tmp_file1[5:]
-        uuid = generate_uuid()
-        cmd = 'rucio upload --rse {0} --scope {1} --guid {2} {3}'.format('MOCK4', scope, uuid, tmp_file1)
-        exitcode, out, err = execute(cmd)
+        item = self.create_and_upload_tmp_file('MOCK4')
+        scope = item['did_scope']
+        name = item['did_name']
+        uuid = item['guid']
 
         # Download specific DID
-        result = download_client.download_dids([{'did': '%s:%s' % (scope, name)}])
-        assert_true(result)
+        result = self.download_client.download_dids([{'did': '%s:%s' % (scope, name)}])
+        nose.tools.assert_true(result)
 
         # Download with wildcard
-        result = download_client.download_dids([{'did': '%s:%s' % (scope, name[:10] + '*')}])
-        assert_true(result)
+        result = self.download_client.download_dids([{'did': '%s:%s' % (scope, name[:-2] + '*')}])
+        nose.tools.assert_true(result)
 
         # Download with filter
-        result = download_client.download_dids([{'filters': {'guid': uuid, 'scope': scope}}])
-        assert_true(result)
+        result = self.download_client.download_dids([{'filters': {'guid': uuid, 'scope': scope}}])
+        nose.tools.assert_true(result)
 
         # Download with wildcard and name
-        result = download_client.download_dids([{'did': '%s:%s' % (scope, '*'), 'filters': {'guid': uuid}}])
-        assert_true(result)
+        result = self.download_client.download_dids([{'did': '%s:%s' % (scope, '*'), 'filters': {'guid': uuid}}])
+        nose.tools.assert_true(result)
diff --git a/lib/rucio/tests/test_replica.py b/lib/rucio/tests/test_replica.py
--- a/lib/rucio/tests/test_replica.py
+++ b/lib/rucio/tests/test_replica.py
@@ -346,6 +346,32 @@ def test_list_replica_with_domain(self):
             assert_in('/i/prefer/the/wan', stdout)
             assert_in('/i/prefer/the/lan', stdout)
 
+    def test_list_replica_with_schemes(self):
+        """ REPLICA (CORE): Add and list file replicas forcing schemes"""
+
+        rc = ReplicaClient()
+
+        rse = 'APERTURE_%s' % rse_name_generator()
+        add_rse(rse)
+
+        add_protocol(rse, {'scheme': 'http',
+                           'hostname': 'http.aperture.com',
+                           'port': 80,
+                           'prefix': '//test/chamber/',
+                           'impl': 'rucio.rse.protocols.gfalv2.Default',
+                           'domains': {
+                               'lan': {'read': 1, 'write': 1, 'delete': 1},
+                               'wan': {'read': 1, 'write': 1, 'delete': 1}}})
+
+        scope = 'mock'
+        name = 'element_%s' % generate_uuid()
+        file_item = {'scope': scope, 'name': name, 'bytes': 1234, 'adler32': 'deadbeef'}
+
+        add_replicas(rse=rse, files=[file_item], account='root')
+
+        replicas = list(rc.list_replicas([{'scope': scope, 'name': name}]))
+        assert_in('http://', replicas[0]['pfns'].keys()[0])
+
     def test_replica_no_site(self):
         """ REPLICA (CORE): Test listing replicas without site attribute """