Repository: rucio/rucio
Issue: Client raising the wrong error

Motivation
----------

From Stephane:

> when I introduce a typo in the RSE name, the error message which I expect 'RSE does not exist' is very complicated : 
> 
> ```
> -bash-4.1$ rucio -v download --rse MWt2_DATADISK mc16_13TeV:HITS.17430965._017098.pool.root.1
> 2019-03-26 10:12:16,014 INFO    Processing 1 item(s) for input
> 2019-03-26 10:12:16,014 DEBUG   Processing item mc16_13TeV:HITS.17430965._017098.pool.root.1
> 2019-03-26 10:12:16,015 DEBUG   RSE-Expression: (MWt2_DATADISK)&istape=False
> 2019-03-26 10:12:16,015 DEBUG   Splitted DID: mc16_13TeV:HITS.17430965._017098.pool.root.1
> 2019-03-26 10:12:16,055 DEBUG   1 DIDs after processing input
> 2019-03-26 10:12:16,056 DEBUG   Processing: {'resolve_archives': True, 'name': 'HITS.17430965._017098.pool.root.1', 'did': 'mc16_13TeV:HITS.17430965._017098.pool.root.1', 'rse': '(MWt2_DATADISK)&istape=False', 'no_subdir': False, 'nrandom': None, 'transfer_timeout': 3600, 'scope': 'mc16_13TeV', 'type': u'FILE', 'force_scheme': None, 'base_dir': '.'}
> 2019-03-26 10:12:16,080 DEBUG   <?xml version="1.0" encoding="UTF-8"?>
> <metalink xmlns="urn:ietf:params:xml:ns:metalink">
> 
> 2019-03-26 10:12:16,092 DEBUG   Traceback (most recent call last):
>   File "/cvmfs/atlas.cern.ch/repo/ATLASLocalRootBase/x86_64/rucio-clients/1.18.5/bin/rucio", line 159, in new_funct
>     return function(*args, **kwargs)
>   File "/cvmfs/atlas.cern.ch/repo/ATLASLocalRootBase/x86_64/rucio-clients/1.18.5/bin/rucio", line 968, in download
>     result = download_client.download_dids(items, args.ndownloader, trace_pattern)
>   File "/cvmfs/atlas.cern.ch/repo/ATLASLocalRootBase/x86_64/rucio-clients/1.18.5/lib/python2.7/site-packages/rucio/client/downloadclient.py", line 291, in download_dids
>     input_items = self._prepare_items_for_download(items)
>   File "/cvmfs/atlas.cern.ch/repo/ATLASLocalRootBase/x86_64/rucio-clients/1.18.5/lib/python2.7/site-packages/rucio/client/downloadclient.py", line 914, in _prepare_items_for_download
>     files_with_pfns = self._parse_list_replica_metalink(metalink_str)
>   File "/cvmfs/atlas.cern.ch/repo/ATLASLocalRootBase/x86_64/rucio-clients/1.18.5/lib/python2.7/site-packages/rucio/client/downloadclient.py", line 996, in _parse_list_replica_metalink
>     raise error
> ExpatError: no element found: line 3, column 0
> 
> 2019-03-26 10:12:16,092 ERROR   no element found: line 3, column 0
> 2019-03-26 10:12:16,093 ERROR   Strange error: No section: 'policy' ```

This should raise `RSENotFound`.

Changed files:
lib/rucio/client/baseclient.py, lib/rucio/web/rest/webpy/v1/replica.py, lib/rucio/client/downloadclient.py

diff --git a/lib/rucio/client/baseclient.py b/lib/rucio/client/baseclient.py
index d1f5ebdd2..3f88e2b0a 100644
--- a/lib/rucio/client/baseclient.py
+++ b/lib/rucio/client/baseclient.py
@@ -357,7 +357,7 @@ class BaseClient(object):
             raise exc_cls(exc_msg)
 
         self.auth_token = result.headers['x-rucio-auth-token']
-        LOG.debug('got new token \\'%s\\'' % self.auth_token)
+        LOG.debug('got new token')
         return True
 
     def __get_token_x509(self):
@@ -418,7 +418,7 @@ class BaseClient(object):
             raise exc_cls(exc_msg)
 
         self.auth_token = result.headers['x-rucio-auth-token']
-        LOG.debug('got new token \\'%s\\'' % self.auth_token)
+        LOG.debug('got new token')
         return True
 
     def __get_token_ssh(self):
@@ -497,7 +497,7 @@ class BaseClient(object):
             raise exc_cls(exc_msg)
 
         self.auth_token = result.headers['x-rucio-auth-token']
-        LOG.debug('got new token \\'%s\\'' % self.auth_token)
+        LOG.debug('got new token')
         return True
 
     def __get_token_gss(self):
@@ -535,7 +535,7 @@ class BaseClient(object):
             raise exc_cls(exc_msg)
 
         self.auth_token = result.headers['x-rucio-auth-token']
-        LOG.debug('got new token \\'%s\\'' % self.auth_token)
+        LOG.debug('got new token')
         return True
 
     def __get_token(self):
@@ -586,7 +586,7 @@ class BaseClient(object):
         except Exception:
             raise
 
-        LOG.debug('read token \\'%s\\' from file' % self.auth_token)
+        LOG.debug('got token from file')
         return True
 
     def __write_token(self):
diff --git a/lib/rucio/client/downloadclient.py b/lib/rucio/client/downloadclient.py
index 7be3ca247..d374d0a96 100644
--- a/lib/rucio/client/downloadclient.py
+++ b/lib/rucio/client/downloadclient.py
@@ -16,7 +16,7 @@
 # - Tomas Javurek <tomasjavurek09@gmail.com>, 2018
 # - Vincent Garonne <vgaronne@gmail.com>, 2018
 # - Joaquin Bogado <jbogado@linti.unlp.edu.ar>, 2018
-# - Nicolo Magini <nicolo.magini@cern.ch>, 2018
+# - Nicolo Magini <nicolo.magini@cern.ch>, 2018-2019
 # - Tobias Wegner <tobias.wegner@cern.ch>, 2018
 # - Hannes Hansen <hannes.jakob.hansen@cern.ch>, 2018-2019
 #
@@ -330,7 +330,9 @@ class DownloadClient:
             dest_file_path = os.path.join(dest_dir_path, did_name)
             item['dest_file_paths'] = [dest_file_path]
             item['temp_file_path'] = '%s.part' % dest_file_path
-            item.setdefault('ignore_checksum', True)
+            options = item.setdefault('merged_options', {})
+            options.setdefault('ignore_checksum', item.pop('ignore_checksum', True))
+            options.setdefault('transfer_timeout', item.pop('transfer_timeout', None))
 
             input_items.append(item)
 
@@ -653,7 +655,7 @@ class DownloadClient:
                 start_time = time.time()
 
                 try:
-                    protocol.get(pfn, temp_file_path, transfer_timeout=item.get('transfer_timeout'))
+                    protocol.get(pfn, temp_file_path, transfer_timeout=item.get('merged_options', {}).get('transfer_timeout'))
                     success = True
                 except Exception as error:
                     logger.debug(error)
@@ -661,7 +663,7 @@ class DownloadClient:
 
                 end_time = time.time()
 
-                if success and not item.get('ignore_checksum', False):
+                if success and not item.get('merged_options', {}).get('ignore_checksum', False):
                     rucio_checksum = item.get('adler32')
                     local_checksum = None
                     if not rucio_checksum:
diff --git a/lib/rucio/web/rest/webpy/v1/replica.py b/lib/rucio/web/rest/webpy/v1/replica.py
index 65f2f7fac..74637fd7e 100755
--- a/lib/rucio/web/rest/webpy/v1/replica.py
+++ b/lib/rucio/web/rest/webpy/v1/replica.py
@@ -112,15 +112,23 @@ class Replicas(RucioController):
             schemes = supported_protocols
 
         try:
-            # first, set the appropriate content type, and stream the header
-            if not metalink:
-                header('Content-Type', 'application/x-json-stream')
-            else:
-                header('Content-Type', 'application/metalink4+xml')
-                yield '<?xml version="1.0" encoding="UTF-8"?>\\n<metalink xmlns="urn:ietf:params:xml:ns:metalink">\\n'
+
+            # we need to call list_replicas before starting to reply
+            # otherwise the exceptions won't be propagated correctly
+            __first = True
 
             # then, stream the replica information
             for rfile in list_replicas(dids=dids, schemes=schemes):
+
+                # in first round, set the appropriate content type, and stream the header
+                if __first:
+                    if not metalink:
+                        header('Content-Type', 'application/x-json-stream')
+                    else:
+                        header('Content-Type', 'application/metalink4+xml')
+                        yield '<?xml version="1.0" encoding="UTF-8"?>\\n<metalink xmlns="urn:ietf:params:xml:ns:metalink">\\n'
+                    __first = False
+
                 client_ip = ctx.env.get('HTTP_X_FORWARDED_FOR')
                 if client_ip is None:
                     client_ip = ctx.ip
@@ -359,12 +367,10 @@ class ListReplicas(RucioController):
             schemes = supported_protocols
 
         try:
-            # first, set the appropriate content type, and stream the header
-            if not metalink:
-                header('Content-Type', 'application/x-json-stream')
-            else:
-                header('Content-Type', 'application/metalink4+xml')
-                yield '<?xml version="1.0" encoding="UTF-8"?>\\n<metalink xmlns="urn:ietf:params:xml:ns:metalink">\\n'
+
+            # we need to call list_replicas before starting to reply
+            # otherwise the exceptions won't be propagated correctly
+            __first = True
 
             # then, stream the replica information
             for rfile in list_replicas(dids=dids, schemes=schemes,
@@ -378,6 +384,16 @@ class ListReplicas(RucioController):
                                        resolve_archives=resolve_archives,
                                        resolve_parents=resolve_parents,
                                        issuer=ctx.env.get('issuer')):
+
+                # in first round, set the appropriate content type, and stream the header
+                if __first:
+                    if not metalink:
+                        header('Content-Type', 'application/x-json-stream')
+                    else:
+                        header('Content-Type', 'application/metalink4+xml')
+                        yield '<?xml version="1.0" encoding="UTF-8"?>\\n<metalink xmlns="urn:ietf:params:xml:ns:metalink">\\n'
+                    __first = False
+
                 if not metalink:
                     yield dumps(rfile, cls=APIEncoder) + '\\n'
                 else: