Repository: rucio/rucio

Issue:
Documentation and cleanup of downloadclient and uploadclient
Motivation
----------
The downloadclient and uploadclient have been already merged to the repository due to prioritised testing. The code still needs to be sanitised and documented.


Discussion:
Files to re-add
:

```(.venv) garvin:rucio garvin$ cat doc/source/api/upload.rst 
--------------
Upload Methods
--------------

.. _upload:
.. automodule:: rucio.client.uploadclient
    :members:
    :undoc-members:
    :show-inheritance:
(.venv) garvin:rucio garvin$ cat doc/source/api/download.rst 
----------------
Download Methods
----------------

.. _download:
.. automodule:: rucio.client.downloadclient
    :members:
    :undoc-members:
    :show-inheritance:
````
just adding them in the repo generates errors/warning when doing `python setup.py build_sphinx`

Resolving patch:
diff --git a/lib/rucio/client/downloadclient.py b/lib/rucio/client/downloadclient.py
--- a/lib/rucio/client/downloadclient.py
+++ b/lib/rucio/client/downloadclient.py
@@ -17,268 +17,292 @@
 # - Vincent Garonne <vgaronne@gmail.com>, 2018
 # - Joaquin Bogado <jbogado@linti.unlp.edu.ar>, 2018
 # - Nicolo Magini <nicolo.magini@cern.ch>, 2018
+# - Tobias Wegner <tobias.wegner@cern.ch>, 2018
 
 import os
 import os.path
 
+import copy
 import logging
 import random
 import time
-import requests
-import json
 import socket
 import signal
 
-from rucio.client.baseclient import BaseClient
-from rucio.common.exception import FileConsistencyMismatch, RSEProtocolNotSupported, RSENotFound, RucioException
-from rucio.common.utils import sizefmt
-from rucio.common.utils import generate_uuid
+from rucio.common.exception import (FileConsistencyMismatch, InputValidationError, NoFilesDownloaded,
+                                    NotAllFilesDownloaded, RSEProtocolNotSupported, RSENotFound, RucioException)
+from rucio.common.utils import detect_client_location, generate_uuid, send_trace, sizefmt
 
 from rucio.client.client import Client
-from Queue import Queue, Empty
+from Queue import Queue, Empty, deque
 from threading import Thread
 from rucio.rse import rsemanager as rsemgr
-from copy import deepcopy
-
-client = Client()
-logging.basicConfig()
-logger = logging.getLogger("user")
-SUCCESS = 0
-FAILURE = 1
-
-DEFAULT_SECURE_PORT = 443
-DEFAULT_PORT = 80
-
-
-class DownloadClient(BaseClient):
-    """ This class cover all functionality related to file uploads into Rucio."""
-
-    BASEURL = '??_downloadclient_??'
-
-    def __init__(self, rucio_host=None, auth_host=None, account=None, ca_cert=None, auth_type=None, creds=None, timeout=None):
-        super(DownloadClient, self).__init__(rucio_host, auth_host, account, ca_cert, auth_type, creds, timeout)
-        self.trace_appid = os.environ.get('RUCIO_TRACE_APPID', None),
-        self.trace_dataset = os.environ.get('RUCIO_TRACE_DATASET', None),
-        self.trace_datasetscope = os.environ.get('RUCIO_TRACE_DATASETSCOPE', None),
-        self.trace_pq = os.environ.get('RUCIO_TRACE_PQ', None),
-        self.trace_taskid = os.environ.get('RUCIO_TRACE_TASKID', None),
-        self.trace_usrdn = os.environ.get('RUCIO_TRACE_USRDN', None)
-
-    def download(self, dids, rse, protocol='srm', pfn=None, nrandom=None, nprocs=None, user_agent='rucio_clients', dir='.', no_subd=False, transfer_timeout=None):
-
-        trace_endpoint = client.host
-        trace_pattern = {'hostname': socket.getfqdn(),
-                         'account': client.account,
-                         'uuid': generate_uuid(),
-                         'eventType': 'download',
-                         'eventVersion': 'api',
-                         'appid': self.trace_appid,
-                         'dataset': self.trace_dataset,
-                         'datasetScope': self.trace_datasetscope,
-                         'pq': self.trace_pq,
-                         'taskid': self.trace_taskid,
-                         'usrdn': self.trace_usrdn}
-
-        # is used account an admin account?
-        account_attributes = [acc for acc in client.list_account_attributes(client.account)]
-        is_admin = False
-        for attr in account_attributes[0]:
-            if attr['key'] == 'admin' and attr['value'] is True:
-                logger.debug('Admin mode enabled')
-                is_admin = True
-                break
 
-        # extend RSE expression to exclude tape RSEs for non-admin accounts
-        rse_expression = rse
-        if not is_admin:
-            rse_expression = 'istape=False'
-            if rse and len(rse.strip()) > 0:
-                rse_expression = '(%s)&istape=False' % rse
-                logger.debug('RSE-Expression: %s' % rse_expression)
 
-        # Extract the scope, name from the did(s)
-        did_list = []
-        for did in dids:
-            try:
-                scope, name = self.extract_scope(did)
-                if name.find('*') > -1:
-                    for dsn in client.list_dids(scope, filters={'name': name}):
-                        did_list.append({'scope': scope, 'name': dsn})
-                else:
-                    did_list.append({'scope': scope, 'name': name})
-            except ValueError as error:
-                raise error
-                return FAILURE
+class DownloadClient:
 
-        if pfn:
-            if not rse:
-                logger.error('--rse option is mandatory in combination with --pfn!')
-                return FAILURE
+    def __init__(self, _client=None, user_agent='rucio_clients', logger=None):
+        """
+        Initialises the basic settings for an DownloadClient object
 
-            if len(dids) > 1:
-                dids = [dids[0]]
-                logger.warning('--pfn option and multiple DIDs given! Only considering first DID...')
+        :param _client: Optional: rucio.client.client.Client object. If None, a new object will be created.
+        :param user_agent: user_agent that is using the download client
+        :param logger: logging.Logger object to use for downloads. If None nothing will be logged.
+        """
+        if not logger:
+            logger = logging.getLogger(__name__).getChild('null')
+            logger.addHandler(logging.NullHandler())
 
-        summary = {}
-        num_files_to_dl = {}
-        input_queue = Queue()
-        output_queue = Queue()
+        self.logger = logger
+        self.is_human_readable = True
+        self.client = _client if _client else Client()
+        self.user_agent = user_agent
 
-        # get replicas for every file of the given dids
-        for arg_did in did_list:
-            arg_didstr = '%s:%s' % (arg_did['scope'], arg_did['name'])
-            summary[arg_didstr] = {}
-
-            # get type of given did; save did if its a dataset
-            files_with_replicas = []
-            if not pfn:
-                try:
-                    did_info = client.get_did(arg_did['scope'], arg_did['name'])
-                    did_type = did_info['type'].upper()
-                    dataset_scope = '' if did_type == 'FILE' else arg_did['scope']
-                    dataset_name = '' if did_type == 'FILE' else arg_did['name']
-                except:
-                    logger.error('Failed to get did info for did %s' % arg_didstr)
-                    return FAILURE
-
-                try:
-                    files_with_replicas = client.list_replicas([arg_did],
-                                                               schemes=None,
-                                                               rse_expression=rse_expression,
-                                                               metalink=None)
-                except:
-                    logger.error('Failed to get list of files with their replicas for DID %s' % arg_didstr)
-                    return FAILURE
-
-                files_with_replicas = [f for f in files_with_replicas]
-                if nrandom:
-                    random.shuffle(files_with_replicas)
-                    files_with_replicas = files_with_replicas[0:nrandom]
+        account_attributes = [acc for acc in self.client.list_account_attributes(self.client.account)]
+        self.is_admin = False
+        for attr in account_attributes[0]:
+            if attr['key'] == 'admin':
+                self.is_admin = attr['value'] is True
+                break
+        if self.is_admin:
+            logger.debug('Admin mode enabled')
+
+        self.trace_tpl = {}
+        self.trace_tpl['hostname'] = socket.getfqdn()
+        self.trace_tpl['account'] = self.client.account
+        self.trace_tpl['eventType'] = 'download'
+        self.trace_tpl['eventVersion'] = 'api'
+
+    def download_pfns(self, items, num_threads=2, trace_custom_fields={}):
+        """
+        Download items with a given PFN. This function can only download files, no datasets.
+
+        :param items: List of dictionaries. Each dictionary describing a file to download. Keys:
+            pfn                 - PFN string of this file
+            did                 - DID string of this file (e.g. 'scope:file.name'). Wildcards are not allowed
+            rse                 - rse name (e.g. 'CERN-PROD_DATADISK'). RSE Expressions are not allowed
+            base_dir            - Optional: Base directory where the downloaded files will be stored. (Default: '.')
+            no_subdir           - Optional: If true, files are written directly into base_dir and existing files are overwritten. (Default: False)
+            ignore_checksum     - Optional: If true, the checksum validation is skipped (for pfn downloads the checksum must be given explicitly). (Default: True)
+            transfer_timeout    - Optional: Timeout time for the download protocols. (Default: None)
+        :param num_threads: Suggestion of number of threads to use for the download. It will be lowered if it's too high.
+        :param trace_custom_fields: Custom key value pairs to send with the traces
+
+        :returns: a list of dictionaries with an entry for each file, containing the input options, the did, and the clientState
+                  clientState can be one of the following: ALREADY_DONE, DONE, FILE_NOT_FOUND, FAIL_VALIDATE, FAILED
+
+        :raises InputValidationError: if one of the input items is in the wrong format
+        :raises NoFilesDownloaded: if no files could be downloaded
+        :raises NotAllFilesDownloaded: if not all files could be downloaded
+        :raises RucioException: if something unexpected went wrong during the download
+        """
+        logger = self.logger
+        trace_custom_fields['uuid'] = generate_uuid()
+
+        logger.info('Processing %d item(s) for input' % len(items))
+        input_items = []
+        for item in items:
+            did_str = item.get('did')
+            pfn = item.get('pfn')
+            rse = item.get('rse')
+
+            if not did_str or not pfn or not rse:
+                logger.debug(item)
+                raise InputValidationError('The keys did, pfn, and rse are mandatory')
+
+            logger.debug('Preparing PFN download of %s (%s) from %s' % (did_str, pfn, rse))
+
+            if '*' in did_str:
+                logger.debug(did_str)
+                raise InputValidationError('Cannot use PFN download with wildcard in DID')
+
+            did_scope, did_name = self._split_did_str(did_str)
+            dest_dir_path = self._prepare_dest_dir(item.get('base_dir', '.'),
+                                                   did_scope, did_name,
+                                                   item.get('no_subdir'))
+
+            item['scope'] = did_scope
+            item['name'] = did_name
+            item['rses'] = {rse: pfn}
+            item['force_scheme'] = pfn.split(':')[0]
+            item['dest_dir_path'] = dest_dir_path
+            item.setdefault('ignore_checksum', True)
+
+            input_items.append(item)
+
+        num_files_in = len(input_items)
+        output_items = self._download_multithreaded(input_items, num_threads, trace_custom_fields)
+        num_files_out = len(output_items)
+
+        if num_files_in != num_files_out:
+            raise RucioException('%d items were in the input queue but only %d are in the output queue' % (num_files_in, num_files_out))
+
+        return self._check_output(output_items)
+
+    def download_dids(self, items, num_threads=2, trace_custom_fields={}):
+        """
+        Download items with given DIDs. This function can also download datasets and wildcarded DIDs.
+
+        :param items: List of dictionaries. Each dictionary describing an item to download. Keys:
+            did                 - DID string of this file (e.g. 'scope:file.name'). Wildcards are not allowed
+            rse                 - Optional: rse name (e.g. 'CERN-PROD_DATADISK') or rse expression from where to download
+            force_scheme        - Optional: force a specific scheme to download this item. (Default: None)
+            base_dir            - Optional: base directory where the downloaded files will be stored. (Default: '.')
+            no_subdir           - Optional: If true, files are written directly into base_dir and existing files are overwritten. (Default: False)
+            nrandom             - Optional: if the DID addresses a dataset, nrandom files will be randomly choosen for download from the dataset
+            ignore_checksum     - Optional: If true, skips the checksum validation between the downloaded file and the rucio catalouge. (Default: False)
+            transfer_timeout    - Optional: Timeout time for the download protocols. (Default: None)
+        :param num_threads: Suggestion of number of threads to use for the download. It will be lowered if it's too high.
+        :param trace_custom_fields: Custom key value pairs to send with the traces
+
+        :returns: a list of dictionaries with an entry for each file, containing the input options, the did, and the clientState
+
+        :raises InputValidationError: if one of the input items is in the wrong format
+        :raises NoFilesDownloaded: if no files could be downloaded
+        :raises NotAllFilesDownloaded: if not all files could be downloaded
+        :raises RucioException: if something unexpected went wrong during the download
+        """
+        logger = self.logger
+        trace_custom_fields['uuid'] = generate_uuid()
+
+        logger.info('Processing %d item(s) for input' % len(items))
+        resolved_items = []
+        for item in items:
+            did_str = item.get('did')
+            if not did_str:
+                raise InputValidationError('The key did is mandatory')
+
+            logger.debug('Processing item %s' % did_str)
+
+            new_item = copy.deepcopy(item)
+
+            # extend RSE expression to exclude tape RSEs for non-admin accounts
+            if not self.is_admin:
+                rse = new_item.get('rse')
+                new_item['rse'] = 'istape=False' if not rse else '(%s)&istape=False' % rse
+                logger.debug('RSE-Expression: %s' % new_item['rse'])
+
+            # resolve any wildcards in the input dids
+            did_scope, did_name = self._split_did_str(did_str)
+            logger.debug('Splitted DID: %s:%s' % (did_scope, did_name))
+            new_item['scope'] = did_scope
+            if '*' in did_name:
+                logger.debug('Resolving wildcarded DID %s' % did_str)
+                for dsn in self.client.list_dids(did_scope, filters={'name': did_name}, type='all'):
+                    logger.debug('%s:%s' % (did_scope, dsn))
+                    new_item['name'] = dsn
+                    new_item['did'] = '%s:%s' % (did_scope, dsn)
+                    resolved_items.append(new_item)
             else:
-                logger.debug('PFN option overrides replica listing')
-                did_type = 'FILE'
-                dataset_scope = ''
-                dataset_name = ''
-                files_with_replicas = [{'bytes': None,
-                                        'adler32': None,
-                                        'scope': arg_did['scope'],
-                                        'name': arg_did['name'],
-                                        'pfns': {pfn: {'rse': rse}},
-                                        'rses': {rse: [pfn]}}]
-
-            num_files_to_dl[arg_didstr] = len(files_with_replicas)
-            for f in files_with_replicas:
-                file_scope = f['scope']
-                file_name = f['name']
-                file_didstr = '%s:%s' % (file_scope, file_name)
-
-                file_exists, dest_dir = self._file_exists(did_type,
-                                                          file_scope,
-                                                          file_name,
-                                                          dir,
-                                                          dsn=dataset_name,
-                                                          no_subdir=no_subd)
-                dest_dir = os.path.abspath(dest_dir)
-
-                if file_exists:
-                    logger.info('File %s already exists locally' % file_didstr)
-
-                    out = {}
-                    out['dataset_scope'] = dataset_scope
-                    out['dataset_name'] = dataset_name
-                    out['scope'] = file_scope
-                    out['name'] = file_name
-                    out['clientState'] = 'ALREADY_DONE'
-                    output_queue.put(out)
-
-                    trace = deepcopy(trace_pattern)
-
-                    if 'datasetScope' not in trace:
-                        trace['datasetScope'] = dataset_scope
-                    if 'dataset' not in trace:
-                        trace['dataset'] = dataset_name
-                    trace.update({'scope': file_scope,
-                                  'filename': file_name,
-                                  'filesize': f['bytes'],
-                                  'transferStart': time.time(),
-                                  'transferEnd': time.time(),
-                                  'clientState': 'ALREADY_DONE'})
-                    self.send_trace(trace, trace_endpoint, user_agent)
-                else:
-                    if not os.path.isdir(dest_dir):
-                        logger.debug('Destination dir not found: %s' % dest_dir)
-                        try:
-                            os.makedirs(dest_dir)
-                        except:
-                            logger.error('Failed to create missing destination directory %s' % dest_dir)
-                            return FAILURE
-                    if no_subd and os.path.isfile('%s/%s' % (dest_dir, file_name)):
-                        # Overwrite the files
-                        logger.debug('Deleteing existing files: %s' % file_name)
-                        os.remove("%s/%s" % (dest_dir, file_name))
-                    f['dataset_scope'] = dataset_scope
-                    f['dataset_name'] = dataset_name
-                    f['dest_dir'] = dest_dir
-                    input_queue.put(f)
+                new_item['name'] = did_name
+                resolved_items.append(new_item)
 
-        try:
-            self.download_rucio(pfn, protocol, input_queue, output_queue, trace_pattern, trace_endpoint, nprocs, user_agent, dir, no_subd, transfer_timeout)
-        except Exception as error:
-            logger.error('Exception during download: %s' % str(error))
+        input_items = []
 
-        while True:
-            try:
-                item = output_queue.get_nowait()
-                output_queue.task_done()
-                ds_didstr = '%s:%s' % (item['dataset_scope'], item['dataset_name'])
-                file_didstr = '%s:%s' % (item['scope'], item['name'])
-
-                if ds_didstr in summary or file_didstr in summary:
-                    if item['dataset_scope'] == '':
-                        summary[file_didstr][file_didstr] = item['clientState']
-                    else:
-                        summary[ds_didstr][file_didstr] = item['clientState']
-                    if item['clientState'] == 'CORRUPTED':
-                        try:
-                            # client.declare_suspicious_file_replicas([item['pfn']], reason='Corrupted')
-                            logger.warning('File %s seems to be corrupted.' % item['pfn'])
-                        except:
-                            logger.warning('File replica %s might be corrupted. Failure to declare it bad to Rucio' % item['pfn'])
-            except Empty:
-                break
-
-    def download_rucio(self, pfn, protocol, input_queue, output_queue, trace_pattern, trace_endpoint, ndownloader, user_agent, dir='.', no_subdir=False, transfer_timeout=None):
+        # get replicas for every file of the given dids
+        logger.debug('%d DIDs after processing input' % len(resolved_items))
+        for item in resolved_items:
+            did_scope = item['scope']
+            did_name = item['name']
+            did_str = item['did']
+
+            logger.debug('Processing: %s' % item)
+
+            # get type of given did
+            did_type = self.client.get_did(did_scope, did_name)['type'].upper()
+            logger.debug('Type: %s' % did_type)
+
+            # get replicas (RSEs) with PFNs for each file (especially if its a dataset)
+            files_with_replicas = self.client.list_replicas([{'scope': did_scope, 'name': did_name}],
+                                                            schemes=item.get('force_scheme'),
+                                                            rse_expression=item.get('rse'),
+                                                            client_location=detect_client_location())
+
+            nrandom = item.get('nrandom')
+            if nrandom:
+                logger.info('Selecting %d random replicas from dataset %s' % (nrandom, did_str))
+                files_with_replicas = list(files_with_replicas)
+                random.shuffle(files_with_replicas)
+                files_with_replicas = files_with_replicas[0:nrandom]
+
+            for file_item in files_with_replicas:
+                file_did_scope = file_item['scope']
+                file_did_name = file_item['name']
+                file_did_str = '%s:%s' % (file_did_scope, file_did_name)
+
+                logger.debug('Queueing file: %s' % file_did_str)
+
+                # put the input options from item into the file item
+                file_item.update(item)
+
+                dest_dir_name = file_did_scope
+                if did_type == 'DATASET':
+                    # if the did is a dataset, scope and name were updated wrongly
+                    file_item['scope'] = file_did_scope
+                    file_item['name'] = file_did_name
+                    file_item['did'] = file_did_str
+                    file_item['dataset_scope'] = did_scope
+                    file_item['dataset_name'] = did_name
+                    dest_dir_name = did_name
+
+                dest_dir_path = self._prepare_dest_dir(item.get('base_dir', '.'),
+                                                       dest_dir_name, file_did_name,
+                                                       item.get('no_subdir'))
+                file_item['dest_dir_path'] = dest_dir_path
+
+                input_items.append(file_item)
+
+        num_files_in = len(input_items)
+        output_items = self._download_multithreaded(input_items, num_threads, trace_custom_fields)
+        num_files_out = len(output_items)
+
+        if num_files_in != num_files_out:
+            raise RucioException('%d items were in the input queue but only %d are in the output queue' % (num_files_in, num_files_out))
+
+        return self._check_output(output_items)
+
+    def _download_multithreaded(self, input_items, num_threads, trace_custom_fields={}):
+        """
+        Starts an appropriate number of threads to download items from the input list.
+        (This function is meant to be used as class internal only)
+
+        :param input_items: list containing the input items to download
+        :param num_threads: suggestion of how many threads should be started
+        :param trace_custom_fields: Custom key value pairs to send with the traces
+
+        :returns: list with output items as dictionaries
+        """
+        logger = self.logger
+
+        num_files = len(input_items)
+        nlimit = 5
+        num_threads = max(1, num_threads)
+        num_threads = min(num_files, num_threads, nlimit)
 
-        total_workers = 1
-        if ndownloader and not pfn:
-            total_workers = ndownloader
-            nlimit = 5
-            if total_workers > nlimit:
-                logger.warning('Cannot use more than %s parallel downloader.' % nlimit)
-                total_workers = nlimit
+        input_queue = Queue()
+        output_queue = Queue()
+        input_queue.queue = deque(input_items)
 
-        total_workers = min(total_workers, input_queue.qsize())
+        if num_threads < 2:
+            logger.info('Using main thread to download %d file(s)' % num_files)
+            self._download_worker(input_queue, output_queue, trace_custom_fields, '')
+            return list(output_queue.queue)
 
-        logger.debug('Starting %d download threads' % total_workers)
+        logger.info('Using %d threads to download %d files' % (num_threads, num_files))
         threads = []
-        for worker in range(total_workers):
-            kwargs = {'pfn': pfn,
-                      'user_agent': user_agent,
-                      'protocol': protocol,
-                      'human': True,
-                      'input_queue': input_queue,
+        for thread_num in range(1, num_threads + 1):
+            log_prefix = 'Thread %s/%s : ' % (thread_num, num_threads)
+            kwargs = {'input_queue': input_queue,
                       'output_queue': output_queue,
-                      'threadnb': worker + 1,
-                      'total_threads': total_workers,
-                      'trace_endpoint': trace_endpoint,
-                      'trace_pattern': trace_pattern,
-                      'transfer_timeout': transfer_timeout}
+                      'trace_custom_fields': trace_custom_fields,
+                      'log_prefix': log_prefix}
             try:
-                thread = Thread(target=self._downloader, kwargs=kwargs)
+                thread = Thread(target=self._download_worker, kwargs=kwargs)
                 thread.start()
                 threads.append(thread)
-            except:
-                logger.warning('Failed to start thread %d' % (worker + 1))
+            except Exception as error:
+                logger.warning('Failed to start thread %d' % thread_num)
+                logger.debug(error)
 
         try:
             logger.debug('Waiting for threads to finish')
@@ -288,206 +312,309 @@ def download_rucio(self, pfn, protocol, input_queue, output_queue, trace_pattern
             logger.warning('You pressed Ctrl+C! Exiting gracefully')
             for thread in threads:
                 thread.kill_received = True
-        logger.debug('All threads finished')
-
-    def _downloader(self, pfn, protocol, human, input_queue, output_queue, user_agent, threadnb, total_threads, trace_endpoint, trace_pattern, transfer_timeout=None):
-
-        rse_dict = {}
-        thread_prefix = 'Thread %s/%s' % (threadnb, total_threads)
+        return list(output_queue.queue)
+
+    def _download_worker(self, input_queue, output_queue, trace_custom_fields, log_prefix):
+        """
+        This function runs as long as there are items in the input queue,
+        downloads them and stores the output in the output queue.
+        (This function is meant to be used as class internal only)
+
+        :param input_queue: queue containing the input items to download
+        :param output_queue: queue where the output items will be stored
+        :param trace_custom_fields: Custom key value pairs to send with the traces
+        :param log_prefix: string that will be put at the beginning of every log message
+        """
+        logger = self.logger
+
+        logger.debug('%sStart processing queued downloads' % log_prefix)
         while True:
             try:
-                file = input_queue.get_nowait()
+                item = input_queue.get_nowait()
             except Empty:
-                return
-
-            dest_dir = file['dest_dir']
-            file_scope = file['scope']
-            file_name = file['name']
-            file_didstr = '%s:%s' % (file_scope, file_name)
-
-            # arguments for rsemgr.download already known
-            dlfile = {}
-            dlfile['name'] = file_name
-            dlfile['scope'] = file_scope
-            dlfile['adler32'] = file['adler32']
-            ignore_checksum = True if pfn else False
-            if pfn:
-                dlfile['pfn'] = pfn
-
-            logger.info('%s : Starting the download of %s' % (thread_prefix, file_didstr))
-            trace = deepcopy(trace_pattern)
-            trace.update({'scope': file_scope,
-                          'filename': file_name,
-                          'datasetScope': file['dataset_scope'],
-                          'dataset': file['dataset_name'],
-                          'filesize': file['bytes']})
-
-            rses = list(file['rses'].keys())
-            if rses == []:
-                logger.warning('%s : File %s has no available replicas. Cannot be downloaded.' % (thread_prefix, file_didstr))
-                trace['clientState'] = 'FILE_NOT_FOUND'
-                self.send_trace(trace, trace_endpoint, user_agent)
-                input_queue.task_done()
+                break
+            try:
+                trace = copy.deepcopy(self.trace_tpl)
+                trace.update(trace_custom_fields)
+                download_result = self._download_item(item, trace, log_prefix)
+                output_queue.put(download_result)
+            except KeyboardInterrupt:
+                logger.warning('You pressed Ctrl+C! Exiting gracefully')
+                os.kill(os.getpgid(), signal.SIGINT)
+                break
+            except Exception as error:
+                logger.error('%sFailed to download item' % log_prefix)
+                logger.debug(error)
+
+    def _download_item(self, item, trace, log_prefix=''):
+        """
+        Downloads the given item and sends traces for success/failure.
+        (This function is meant to be used as class internal only)
+
+        :param item: dictionary that describes the item to download
+        :param trace: dictionary representing a pattern of trace that will be send
+        :param log_prefix: string that will be put at the beginning of every log message
+
+        :returns: dictionary with all attributes from the input item and a clientState attribute
+        """
+        logger = self.logger
+
+        did_scope = item['scope']
+        did_name = item['name']
+        did_str = '%s:%s' % (did_scope, did_name)
+
+        logger.info('%sPreparing download of %s' % (log_prefix, did_str))
+
+        trace['scope'] = did_scope
+        trace['filename'] = did_name
+        trace.setdefault('dataset_scope', item.get('dataset_scope', ''))
+        trace.setdefault('dataset', item.get('dataset_name', ''))
+        trace.setdefault('filesize', item.get('bytes'))
+
+        # if file already exists, set state, send trace, and return
+        dest_dir_path = item['dest_dir_path']
+        dest_file_path = os.path.join(dest_dir_path, did_name)
+        if os.path.isfile(dest_file_path):
+            logger.info('%sFile exists already locally: %s' % (log_prefix, did_str))
+            item['clientState'] = 'ALREADY_DONE'
+
+            trace['transferStart'] = time.time()
+            trace['transferEnd'] = time.time()
+            trace['clientState'] = 'ALREADY_DONE'
+            send_trace(trace, self.client.host, self.user_agent)
+            return item
+
+        # check if file has replicas
+        rse_names = list(item['rses'].keys())
+        if not len(rse_names):
+            logger.warning('%sFile %s has no available replicas. Cannot be downloaded' % (log_prefix, did_str))
+            item['clientState'] = 'FILE_NOT_FOUND'
+
+            trace['clientState'] = 'FILE_NOT_FOUND'
+            send_trace(trace, self.client.host, self.user_agent)
+            return item
+
+        # list_replicas order is: best rse at [0]
+        rse_names.reverse()
+
+        logger.debug('%sPotential sources: %s' % (log_prefix, str(rse_names)))
+
+        success = False
+        # retry with different rses if one is not available or fails
+        while not success and len(rse_names):
+            rse_name = rse_names.pop()
+            try:
+                rse = rsemgr.get_rse_info(rse_name)
+            except RSENotFound:
+                logger.warning('%sCould not get info of RSE %s' % (log_prefix, rse_name))
+                continue
+
+            if not rse['availability_read']:
+                logger.info('%s%s is blacklisted for reading' % (log_prefix, rse_name))
                 continue
-            random.shuffle(rses)
 
-            logger.debug('%s : Potential sources : %s' % (thread_prefix, str(rses)))
-            success = False
-            while not success and len(rses):
-                rse_name = rses.pop()
-                if rse_name not in rse_dict:
+            force_scheme = item.get('force_scheme')
+            try:
+                protocols = rsemgr.get_protocols_ordered(rse, operation='read', scheme=force_scheme)
+                protocols.reverse()
+            except RSEProtocolNotSupported as error:
+                logger.info('%sThe protocol specfied (%s) is not supported by %s' % (log_prefix, force_scheme, rse_name))
+                logger.debug(error)
+                continue
+
+            logger.debug('%sPotential protocol(s) read: %s' % (log_prefix, protocols))
+
+            trace['remoteSite'] = rse_name
+            trace['clientState'] = 'DOWNLOAD_ATTEMPT'
+
+            # retry with different protocols on the given rse
+            while not success and len(protocols):
+                protocol = protocols.pop()
+                cur_scheme = protocol['scheme']
+                trace['protocol'] = cur_scheme
+
+                logger.info('%sTrying to download with %s from %s: %s ' % (log_prefix, cur_scheme, rse_name, did_str))
+
+                attempt = 0
+                retries = 2
+                # do some retries with the same rse and protocol if the download fails
+                while not success and attempt < retries:
+                    attempt += 1
+                    item['attemptnr'] = attempt
+
                     try:
-                        rse_dict[rse_name] = rsemgr.get_rse_info(rse_name)
-                    except RSENotFound:
-                        logger.warning('%s : Could not get info of RSE %s' % (thread_prefix, rse_name))
-                        continue
-
-                rse = rse_dict[rse_name]
-
-                if not rse['availability_read']:
-                    logger.info('%s : %s is blacklisted for reading' % (thread_prefix, rse_name))
-                    continue
-
-                try:
-                    if pfn:
-                        protocols = [rsemgr.select_protocol(rse, operation='read', scheme=pfn.split(':')[0])]
-                    else:
-                        protocols = rsemgr.get_protocols_ordered(rse, operation='read', scheme=protocol)
-                        protocols.reverse()
-                except RSEProtocolNotSupported as error:
-                    logger.info('%s : The protocol specfied (%s) is not supported by %s' % (thread_prefix, protocol, rse_name))
-                    logger.debug(error)
-                    continue
-                logger.debug('%s : %d possible protocol(s) for read' % (thread_prefix, len(protocols)))
-                trace['remoteSite'] = rse_name
-                trace['clientState'] = 'DOWNLOAD_ATTEMPT'
-
-                while not success and len(protocols):
-                    protocol_retry = protocols.pop()
-                    logger.debug('%s : Trying protocol %s at %s' % (thread_prefix, protocol_retry['scheme'], rse_name))
-                    trace['protocol'] = protocol_retry['scheme']
-                    out = {}
-                    out['dataset_scope'] = file['dataset_scope']
-                    out['dataset_name'] = file['dataset_name']
-                    out['scope'] = file_scope
-                    out['name'] = file_name
-
-                    attempt = 0
-                    retries = 2
-                    while not success and attempt < retries:
-                        attempt += 1
-                        out['attemptnr'] = attempt
-
-                        logger.info('%s : File %s trying from %s' % (thread_prefix, file_didstr, rse_name))
+
+                        start_time = time.time()
+                        rsemgr.download(rse,
+                                        files=item,
+                                        dest_dir=dest_dir_path,
+                                        force_scheme=cur_scheme,
+                                        ignore_checksum=item.get('ignore_checksum', False),
+                                        transfer_timeout=item.get('transfer_timeout'))
+                        end_time = time.time()
+
+                        trace['transferStart'] = start_time
+                        trace['transferEnd'] = end_time
+                        trace['clientState'] = 'DONE'
+                        item['clientState'] = 'DONE'
+                        success = True
+                    except FileConsistencyMismatch as error:
+                        logger.warning(str(error))
                         try:
-                            trace['transferStart'] = time.time()
-
-                            rsemgr.download(rse,
-                                            files=[dlfile],
-                                            dest_dir=dest_dir,
-                                            force_scheme=protocol_retry['scheme'],
-                                            ignore_checksum=ignore_checksum,
-                                            transfer_timeout=transfer_timeout)
-
-                            trace['transferEnd'] = time.time()
-                            trace['clientState'] = 'DONE'
-                            out['clientState'] = 'DONE'
-                            success = True
-                            output_queue.put(out)
-                            logger.info('%s : File %s successfully downloaded from %s' % (thread_prefix, file_didstr, rse_name))
-                        except KeyboardInterrupt:
-                            logger.warning('You pressed Ctrl+C! Exiting gracefully')
-                            os.kill(os.getpgid(), signal.SIGINT)
-                            return
-                        except FileConsistencyMismatch as error:
-                            logger.warning(str(error))
-                            try:
+                            pfn = item.get('pfn')
+                            if not pfn:
                                 pfns_dict = rsemgr.lfns2pfns(rse,
-                                                             lfns=[{'name': file_name, 'scope': file_scope}],
+                                                             lfns={'name': did_name, 'scope': did_scope},
                                                              operation='read',
-                                                             scheme=protocol)
-                                pfn = pfns_dict[file_didstr]
-
-                                out['clientState'] = 'CORRUPTED'
-                                out['pfn'] = pfn
-                                output_queue.put(out)
-                            except Exception as error:
-                                logger.debug('%s : %s' % (thread_prefix, str(error)))
-                            trace['clientState'] = 'FAIL_VALIDATE'
-                            logger.debug('%s : Failed attempt %s/%s' % (thread_prefix, attempt, retries))
+                                                             scheme=cur_scheme)
+                                pfn = pfns_dict[did_str]
+
+                            corrupted_item = copy.deepcopy(item)
+                            corrupted_item['clientState'] = 'FAIL_VALIDATE'
+                            corrupted_item['pfn'] = pfn
+                            # self.corrupted_files.append(corrupted_item)
                         except Exception as error:
-                            logger.warning(str(error))
-                            trace['clientState'] = str(type(error).__name__)
-                            logger.debug('%s : Failed attempt %s/%s' % (thread_prefix, attempt, retries))
-
-                    self.send_trace(trace, trace_endpoint, user_agent, threadnb=threadnb, total_threads=total_threads)
-
-            if success:
-                duration = round(trace['transferEnd'] - trace['transferStart'], 2)
-                if pfn:
-                    logger.info('%s : File %s successfully downloaded in %s seconds' % (thread_prefix, file_didstr, duration))
-                else:
-                    logger.info('%s : File %s successfully downloaded. %s in %s seconds = %s MBps' % (thread_prefix,
-                                                                                                      file_didstr,
-                                                                                                      sizefmt(file['bytes'], human),
-                                                                                                      duration,
-                                                                                                      round((file['bytes'] / duration) * 1e-6, 2)))
-            else:
-                logger.error('%s : Cannot download file %s' % (thread_prefix, file_didstr))
-
-            input_queue.task_done()
-
-    def extract_scope(self, did):
-        # Try to extract the scope from the DSN
-        if did.find(':') > -1:
-            if len(did.split(':')) > 2:
-                raise RucioException('Too many colons. Cannot extract scope and name')
-            scope, name = did.split(':')[0], did.split(':')[1]
-            if name.endswith('/'):
-                name = name[:-1]
-            return scope, name
+                            logger.debug('%s%s' % (log_prefix, str(error)))
+                        trace['clientState'] = 'FAIL_VALIDATE'
+                    except Exception as error:
+                        logger.warning(str(error))
+                        trace['clientState'] = str(type(error).__name__)
+
+                    if not success:
+                        logger.debug('%sFailed attempt %s/%s' % (log_prefix, attempt, retries))
+
+                    send_trace(trace, self.client.host, self.user_agent)
+
+        if not success:
+            logger.error('%sFailed to download file %s' % (log_prefix, did_str))
+            item['clientState'] = 'FAILED'
+            return item
+
+        duration = round(end_time - start_time, 2)
+        size = item.get('bytes')
+        size_str = sizefmt(size, self.is_human_readable)
+        if size and duration:
+            rate = round((size / duration) * 1e-6, 2)
+            logger.info('%sFile %s successfully downloaded. %s in %s seconds = %s MBps' % (log_prefix, did_str, size_str, duration, rate))
         else:
-            scope = did.split('.')[0]
-            if did.startswith('user') or did.startswith('group'):
-                scope = ".".join(did.split('.')[0:2])
-            if did.endswith('/'):
-                did = did[:-1]
-            return scope, did
-
-    def _file_exists(self, type, scope, name, directory, dsn=None, no_subdir=False):
-        file_exists = False
-        dest_dir = None
-        if no_subdir:
-            dest_dir = '%s' % (directory)
+            logger.info('%sFile %s successfully downloaded in %s seconds' % (log_prefix, did_str, duration))
+        return item
+
+    def download(self, dids, rse, protocol=None, pfn=None, nrandom=None, nprocs=3, user_agent='rucio_clients', dir='.', no_subd=False, transfer_timeout=None):
+        """
+        OBSOLETE! This function is kept for compability reasons and will be removed in a future release!
+        """
+        item_tpl = {'rse': rse,
+                    'force_scheme': protocol,
+                    'nrandom': nrandom,
+                    'base_dir': dir,
+                    'no_subdir': no_subd,
+                    'transfer_timeout': transfer_timeout}
+        if pfn:
+            item_tpl['pfn'] = pfn
+        input_items = []
+        for did in dids:
+            item = {}
+            item.update(item_tpl)
+            item['did'] = did
+            input_items.append(item)
+
+        trace_pattern = {'appid': os.environ.get('RUCIO_TRACE_APPID', None),
+                         'dataset': os.environ.get('RUCIO_TRACE_DATASET', None),
+                         'datasetScope': os.environ.get('RUCIO_TRACE_DATASETSCOPE', None),
+                         'pq': os.environ.get('RUCIO_TRACE_PQ', None),
+                         'taskid': os.environ.get('RUCIO_TRACE_TASKID', None),
+                         'usrdn': os.environ.get('RUCIO_TRACE_USRDN', None)}
+
+        if pfn:
+            return self.download_pfns(input_items, nprocs, trace_pattern)
         else:
-            if type != 'FILE':
-                dest_dir = '%s/%s' % (directory, dsn)
-                if os.path.isfile('%s/%s' % (dest_dir, name)):
-                    file_exists = True
-            else:
-                dest_dir = '%s/%s' % (directory, scope)
-                if os.path.isfile('%s/%s' % (dest_dir, name)):
-                    file_exists = True
-        return file_exists, dest_dir
+            return self.download_dids(input_items, nprocs, trace_pattern)
+
+    def _split_did_str(self, did_str):
+        """
+        Splits a given DID string (e.g. 'scope1:name.file') into its scope and name part
+        (This function is meant to be used as class internal only)
+
+        :param did_str: the DID string that will be splitted
+
+        :returns: the scope- and name part of the given DID
+
+        :raises InputValidationError: if the given DID string is not valid
+        """
+        did = did_str.split(':')
+        if len(did) == 2:
+            did_scope = did[0]
+            did_name = did[1]
+        elif len(did) == 1:
+            did = did_str.split('.')
+            did_scope = did[0]
+            if did_scope == 'user' or did_scope == 'group':
+                did_scope = '%s.%s' % (did[0], did[1])
+            did_name = did_str
+        else:
+            raise InputValidationError('%s is not a valid DID. To many colons.' % did_str)
+
+        if did_name.endswith('/'):
+            did_name = did_name[:-1]
+
+        return did_scope, did_name
 
-    def send_trace(self, trace, trace_endpoint, user_agent, retries=5, threadnb=None, total_threads=None):
+    def _prepare_dest_dir(self, base_dir, dest_dir_name, file_name, no_subdir):
+        """
+        Builds the final destination path for a file and:
+            1. deletes existing files if no_subdir was given
+            2. creates the destination directory if it's not existent
+        (This function is meant to be used as class internal only)
 
-        if user_agent.startswith('pilot'):
-            logger.debug('pilot detected - not sending trace')
-            return 0
+        :param base_dir: base directory part
+        :param dest_dir_name: name of the destination directory
+        :param file_name: name of the file that will be downloaded
+        :param no_subdir: if no subdirectory should be created
+
+        :returns: the absolut path of the destination directory
+        """
+        dest_dir_path = os.path.abspath(base_dir)
+        # if no subdirectory is used, existing files will be overwritten
+        if no_subdir:
+            dest_file_path = os.path.join(dest_dir_path, file_name)
+            if os.path.isfile(dest_file_path):
+                self.logger.debug('Deleting existing file: %s' % dest_file_path)
+                os.remove(dest_file_path)
         else:
-            if threadnb is not None and total_threads is not None:
-                logger.debug('Thread %s/%s : sending trace' % (threadnb, total_threads))
+            dest_dir_path = os.path.join(dest_dir_path, dest_dir_name)
+
+        if not os.path.isdir(dest_dir_path):
+            os.makedirs(dest_dir_path)
+
+        return dest_dir_path
+
+    def _check_output(self, output_items):
+        """
+        Checks if all files were successfully downloaded
+        (This function is meant to be used as class internal only)
+
+        :param output_items: list of dictionaries describing the downloaded files
+
+        :returns: output_items list
+
+        :raises NoFilesDownloaded:
+        :raises NotAllFilesDownloaded:
+        """
+        success_states = ['ALREADY_DONE', 'DONE']
+        # failure_states = ['FILE_NOT_FOUND', 'FAIL_VALIDATE', 'FAILED']
+        num_successful = 0
+        num_failed = 0
+        for item in output_items:
+            clientState = item.get('clientState', 'FAILED')
+            if clientState in success_states:
+                num_successful += 1
             else:
-                logger.debug('sending trace')
+                num_failed += 1
 
-        for dummy in range(retries):
-            try:
-                requests.post(trace_endpoint + '/traces/', verify=False, data=json.dumps(trace))
-                return 0
-            except Exception as error:
-                if threadnb is not None and total_threads is not None:
-                    logger.debug('Thread %s/%s : %s' % (threadnb, total_threads, error))
-                else:
-                    logger.debug(error)
-        return 1
+        if num_successful == 0:
+            raise NoFilesDownloaded()
+        elif num_failed > 0:
+            raise NotAllFilesDownloaded()
+
+        return output_items
diff --git a/lib/rucio/client/uploadclient.py b/lib/rucio/client/uploadclient.py
--- a/lib/rucio/client/uploadclient.py
+++ b/lib/rucio/client/uploadclient.py
@@ -39,11 +39,17 @@
 
 
 class UploadClient:
-    """ This class cover all functionality related to file uploads into Rucio."""
 
     def __init__(self, _client=None, user_agent='rucio_clients', logger=None):
+        """
+        Initialises the basic settings for an UploadClient object
+
+        :param _client:     - Optional: rucio.client.client.Client object. If None, a new object will be created.
+        :param user_agent:  - user_agent that is using the upload client
+        :param logger:      - logging.Logger object to use for uploads. If None nothing will be logged.
+        """
         if not logger:
-            logger = logging.getLogger('rucio_uploadclient')
+            logger = logging.getLogger(__name__).getChild('null')
             logger.addHandler(logging.NullHandler())
 
         self.logger = logger
@@ -60,129 +66,35 @@ def __init__(self, _client=None, user_agent='rucio_clients', logger=None):
         self.trace['eventType'] = 'upload'
         self.trace['eventVersion'] = version.RUCIO_VERSION[0]
 
-    def get_file_guid(self, file):
-        guid = file.get('guid')
-        if not guid and 'pool.root' in file['basename'].lower() and not file.get('no_register'):
-            status, output, err = execute('pool_extractFileIdentifier %s' % file['path'])
-            if status != 0:
-                msg = 'Trying to upload ROOT files but pool_extractFileIdentifier tool can not be found.\n'
-                msg += 'Setup your ATHENA environment and try again.'
-                raise RucioException(msg)
-            try:
-                guid = output.splitlines()[-1].split()[0].replace('-', '').lower()
-            except Exception:
-                raise RucioException('Error extracting GUID from ouput of pool_extractFileIdentifier')
-        elif guid:
-            guid = guid.replace('-', '')
-        else:
-            guid = generate_uuid()
-        return guid
-
-    def collect_file_info(self, filepath, settings):
-        file = copy.deepcopy(settings)
-        file['path'] = filepath
-        file['dirname'] = os.path.dirname(filepath)
-        file['basename'] = os.path.basename(filepath)
-
-        file['bytes'] = os.stat(filepath).st_size
-        file['adler32'] = adler32(filepath)
-        file['md5'] = md5(filepath)
-        file['meta'] = {'guid': self.get_file_guid(file)}
-        file['state'] = 'C'
-        file.setdefault('did_scope', self.default_file_scope)
-        file.setdefault('did_name', file['basename'])
-        file.setdefault('lifetime', None)
-
-        return file
-
-    def collect_and_validate_file_info(self, sources_with_settings):
-        logger = self.logger
-        files = []
-        for settings in sources_with_settings:
-            path = settings.get('path')
-            pfn = settings.get('pfn')
-            if not path:
-                logger.warning('Skipping source entry because the key "path" is missing')
-                continue
-            if not settings.get('rse'):
-                logger.warning('Skipping file %s because no rse was given' % path)
-                continue
-            if pfn:
-                if settings.get('no_register'):
-                    logger.warning('Upload with given pfn implies that no_register is True')
-                    settings['no_register'] = True
-                scheme = settings.get('scheme')
-                pfn_scheme = pfn.split(':')[0]
-                if scheme and scheme != pfn_scheme:
-                    logger.warning('PFN scheme (%s) overrides given scheme (%s)' % (pfn_scheme, scheme))
-                    scheme = pfn_scheme
-                settings['scheme'] = pfn_scheme
-
-            if os.path.isdir(path):
-                dname, subdirs, fnames = next(os.walk(path))
-                for fname in fnames:
-                    file = self.collect_file_info(os.path.join(dname, fname), settings)
-                    files.append(file)
-                if not len(fnames) and not len(subdirs):
-                    logger.warning('Skipping %s because it is empty.' % dname)
-                elif not len(fnames):
-                    logger.warning('Skipping %s because it has no files in it. Subdirectories are not supported.' % dname)
-            elif os.path.isfile(path):
-                file = self.collect_file_info(path, settings)
-                files.append(file)
-            else:
-                logger.warning('No such file or directory: %s' % path)
-
-        if not len(files):
-            raise InputValidationError('No valid input files given')
-
-        return files
-
-    def convert_file_for_api(self, file):
-        replica = {}
-        replica['scope'] = file['did_scope']
-        replica['name'] = file['did_name']
-        replica['bytes'] = file['bytes']
-        replica['adler32'] = file['adler32']
-        replica['md5'] = file['md5']
-        replica['meta'] = file['meta']
-        replica['state'] = file['state']
-        return replica
-
     def upload(self, sources_with_settings, summary_file_path=None):
         """
-        List of dictionaries of file descriptions. None means optional
-        [{'path': 'file1',
-          'rse': 'rse_name1',
-          'did_scope': None,
-          'did_name': None,
-          'dataset_name': None,
-          'dataset_scope': None,
-          'scheme': None,
-          'pfn': None,
-          'no_register': None,
-          'lifetime': None },
-
-         {'path': 'file2',
-          'rse': 'rse_name2',
-          'did_scope': None,
-          'did_name': None,
-          'dataset_name': None,
-          'dataset_scope': None,
-          'scheme': None,
-          'pfn': None,
-          'no_register': None,
-          'lifetime': None }]
-
-          raises InputValidationError
-          raises RSEBlacklisted
+
+        :param items: List of dictionaries. Each dictionary describing a file to upload. Keys:
+            path            - path of the file that will be uploaded
+            rse             - rse name (e.g. 'CERN-PROD_DATADISK') where to upload the file
+            did_scope       - Optional: custom did scope (Default: user.<account>)
+            did_name        - Optional: custom did name (Default: name of the file)
+            dataset_scope   - Optional: custom dataset scope
+            dataset_name    - Optional: custom dataset name
+            force_scheme    - Optional: force a specific scheme (if PFN upload this will be overwritten) (Default: None)
+            pfn             - Optional: use a given PFN (this sets no_register to True)
+            no_register     - Optional: if True, the file will not be registered in the rucio catalogue
+            lifetime        - Optional: the lifetime of the file after it was uploaded
+        :param summary_file_path: Optional: a path where a summary in form of a json file will be stored
+
+        :returns: 0 on success
+
+        :raises InputValidationError: if any input arguments are in a wrong format
+        :raises RSEBlacklisted: if a given RSE is not available for writing
+        :raise NoFilesUploaded: if no files were successfully uploaded
+        :raise NotAllFilesUploaded: if not all files were successfully uploaded
         """
         logger = self.logger
 
         self.trace['uuid'] = generate_uuid()
 
         # check given sources, resolve dirs into files, and collect meta infos
-        files = self.collect_and_validate_file_info(sources_with_settings)
+        files = self._collect_and_validate_file_info(sources_with_settings)
 
         # check if RSE of every file is available for writing
         # and cache rse settings
@@ -217,7 +129,7 @@ def upload(self, sources_with_settings, summary_file_path=None):
 
             no_register = file.get('no_register')
             pfn = file.get('pfn')
-            scheme = file.get('scheme')
+            force_scheme = file.get('force_scheme')
 
             self.trace['scope'] = file['did_scope']
             self.trace['datasetScope'] = file.get('dataset_scope', '')
@@ -250,7 +162,7 @@ def upload(self, sources_with_settings, summary_file_path=None):
                     # TODO: Need to check the rules thing!!
                     logger.info("Dataset %s already exists" % dataset_did_str)
 
-            replica_for_api = self.convert_file_for_api(file)
+            replica_for_api = self._convert_file_for_api(file)
             try:
                 # if the remote checksum is different this did must not be used
                 meta = self.client.get_metadata(file_scope, file_name)
@@ -275,13 +187,14 @@ def upload(self, sources_with_settings, summary_file_path=None):
 
             # if file already exists on RSE we're done
             if not rsemgr.exists(rse_settings, file_did):
-                protocols = rsemgr.get_protocols_ordered(rse_settings=rse_settings, operation='write', scheme=scheme)
+                protocols = rsemgr.get_protocols_ordered(rse_settings=rse_settings, operation='write', scheme=force_scheme)
                 protocols.reverse()
                 success = False
                 summary = []
                 while not success and len(protocols):
                     protocol = protocols.pop()
-                    logger.info('Trying upload to %s with protocol %s' % (rse, protocol['scheme']))
+                    cur_scheme = protocol['scheme']
+                    logger.info('Trying upload with %s to %s' % (cur_scheme, rse))
                     lfn = {}
                     lfn['filename'] = file['basename']
                     lfn['scope'] = file['did_scope']
@@ -289,13 +202,13 @@ def upload(self, sources_with_settings, summary_file_path=None):
                     lfn['adler32'] = file['adler32']
                     lfn['filesize'] = file['bytes']
 
-                    self.trace['protocol'] = protocol['scheme']
+                    self.trace['protocol'] = cur_scheme
                     self.trace['transferStart'] = time.time()
                     try:
                         state = rsemgr.upload(rse_settings=rse_settings,
                                               lfns=lfn,
                                               source_dir=file['dirname'],
-                                              force_scheme=protocol['scheme'],
+                                              force_scheme=cur_scheme,
                                               force_pfn=pfn,
                                               transfer_timeout=file.get('transfer_timeout'))
                         success = True
@@ -331,7 +244,7 @@ def upload(self, sources_with_settings, summary_file_path=None):
                         logger.warning(error)
 
                 logger.info('Setting replica state to available')
-                replica_for_api = self.convert_file_for_api(file)
+                replica_for_api = self._convert_file_for_api(file)
                 self.client.update_replicas_states(rse, files=[replica_for_api])
 
         if summary_file_path:
@@ -356,3 +269,125 @@ def upload(self, sources_with_settings, summary_file_path=None):
         elif num_succeeded != len(files):
             raise NotAllFilesUploaded()
         return 0
+
+    def _get_file_guid(self, file):
+        """
+        Get the guid of a file, trying different strategies
+        (This function is meant to be used as class internal only)
+
+        :param file: dictionary describing the file
+
+        :returns: the guid
+        """
+        guid = file.get('guid')
+        if not guid and 'pool.root' in file['basename'].lower() and not file.get('no_register'):
+            status, output, err = execute('pool_extractFileIdentifier %s' % file['path'])
+            if status != 0:
+                msg = 'Trying to upload ROOT files but pool_extractFileIdentifier tool can not be found.\n'
+                msg += 'Setup your ATHENA environment and try again.'
+                raise RucioException(msg)
+            try:
+                guid = output.splitlines()[-1].split()[0].replace('-', '').lower()
+            except Exception:
+                raise RucioException('Error extracting GUID from ouput of pool_extractFileIdentifier')
+        elif guid:
+            guid = guid.replace('-', '')
+        else:
+            guid = generate_uuid()
+        return guid
+
+    def _collect_file_info(self, filepath, settings):
+        """
+        Collects infos (e.g. size, checksums, etc.) about the file and
+        returns them as a dictionary
+        (This function is meant to be used as class internal only)
+
+        :param filepath: path where the file is stored
+        :param settings: input options for the given file
+
+        :returns: a dictionary containing all collected info and the input options
+        """
+        file = copy.deepcopy(settings)
+        file['path'] = filepath
+        file['dirname'] = os.path.dirname(filepath)
+        file['basename'] = os.path.basename(filepath)
+
+        file['bytes'] = os.stat(filepath).st_size
+        file['adler32'] = adler32(filepath)
+        file['md5'] = md5(filepath)
+        file['meta'] = {'guid': self._get_file_guid(file)}
+        file['state'] = 'C'
+        file.setdefault('did_scope', self.default_file_scope)
+        file.setdefault('did_name', file['basename'])
+        file.setdefault('lifetime', None)
+
+        return file
+
+    def _collect_and_validate_file_info(self, sources_with_settings):
+        """
+        Checks if there are any inconsistencies within the given input
+        options and stores the output of _collect_file_info for every file
+        (This function is meant to be used as class internal only)
+
+        :param filepath: list of dictionaries with all input files and options
+
+        :returns: a list of dictionaries containing all descriptions of the files to upload
+
+        :raises InputValidationError: if an input option has a wrong format
+        """
+        logger = self.logger
+        files = []
+        for settings in sources_with_settings:
+            path = settings.get('path')
+            pfn = settings.get('pfn')
+            if not path:
+                logger.warning('Skipping source entry because the key "path" is missing')
+                continue
+            if not settings.get('rse'):
+                logger.warning('Skipping file %s because no rse was given' % path)
+                continue
+            if pfn:
+                if settings.get('no_register'):
+                    logger.warning('Upload with given pfn implies that no_register is True')
+                    settings['no_register'] = True
+                settings['force_scheme'] = pfn.split(':')[0]
+
+            if os.path.isdir(path):
+                dname, subdirs, fnames = next(os.walk(path))
+                for fname in fnames:
+                    file = self._collect_file_info(os.path.join(dname, fname), settings)
+                    files.append(file)
+                if not len(fnames) and not len(subdirs):
+                    logger.warning('Skipping %s because it is empty.' % dname)
+                elif not len(fnames):
+                    logger.warning('Skipping %s because it has no files in it. Subdirectories are not supported.' % dname)
+            elif os.path.isfile(path):
+                file = self._collect_file_info(path, settings)
+                files.append(file)
+            else:
+                logger.warning('No such file or directory: %s' % path)
+
+        if not len(files):
+            raise InputValidationError('No valid input files given')
+
+        return files
+
+    def _convert_file_for_api(self, file):
+        """
+        Creates a new dictionary that contains only the values
+        that are needed for the upload with the correct keys
+        (This function is meant to be used as class internal only)
+
+        :param file: dictionary describing a file to upload
+
+        :returns: dictionary containing not more then the needed values for the upload
+        """
+        replica = {}
+        replica['scope'] = file['did_scope']
+        replica['name'] = file['did_name']
+        replica['bytes'] = file['bytes']
+        replica['adler32'] = file['adler32']
+        replica['md5'] = file['md5']
+        replica['meta'] = file['meta']
+        replica['state'] = file['state']
+        return replica


Test patch: