Repository: rucio/rucio

Issue:
CLI/python Clients - Recursive Upload of an input folder
Motivation
----------
Currently is possible to upload the files contained in a folder. In order to facilitate the upload tasks we are interested in a recursive exploration of a given input folder. The goal is to replicate the input folder structure by creating sub-datasets then performing normal upload for the found files attaching them to the correct sub-datasets.


Modification
------------
Enhance the "folder" input management of the upload command in the CLI/python Clients.


CLI/python Clients - Recursive Upload of an input folder
Motivation
----------
Currently is possible to upload the files contained in a folder. In order to facilitate the upload tasks we are interested in a recursive exploration of a given input folder. The goal is to replicate the input folder structure by creating sub-datasets then performing normal upload for the found files attaching them to the correct sub-datasets.


Modification
------------
Enhance the "folder" input management of the upload command in the CLI/python Clients.




Discussion:

Resolving patch:
diff --git a/lib/rucio/client/uploadclient.py b/lib/rucio/client/uploadclient.py
--- a/lib/rucio/client/uploadclient.py
+++ b/lib/rucio/client/uploadclient.py
@@ -111,6 +111,7 @@ def upload(self, items, summary_file_path=None, traces_copy_out=None):
             lifetime              - Optional: the lifetime of the file after it was uploaded
             transfer_timeout      - Optional: time after the upload will be aborted
             guid                  - Optional: guid of the file
+            recursive             - Optional: if set, parses the folder structure recursively into collections
         :param summary_file_path: Optional: a path where a summary in form of a json file will be stored
         :param traces_copy_out: reference to an external list, where the traces should be uploaded
 
@@ -490,6 +491,7 @@ def _collect_and_validate_file_info(self, items):
         for item in items:
             path = item.get('path')
             pfn = item.get('pfn')
+            recursive = item.get('recursive')
             if not path:
                 logger(logging.WARNING, 'Skipping source entry because the key "path" is missing')
                 continue
@@ -498,7 +500,7 @@ def _collect_and_validate_file_info(self, items):
                 continue
             if pfn:
                 item['force_scheme'] = pfn.split(':')[0]
-            if os.path.isdir(path):
+            if os.path.isdir(path) and not recursive:
                 dname, subdirs, fnames = next(os.walk(path))
                 for fname in fnames:
                     file = self._collect_file_info(os.path.join(dname, fname), item)
@@ -507,9 +509,13 @@ def _collect_and_validate_file_info(self, items):
                     logger(logging.WARNING, 'Skipping %s because it is empty.' % dname)
                 elif not len(fnames):
                     logger(logging.WARNING, 'Skipping %s because it has no files in it. Subdirectories are not supported.' % dname)
-            elif os.path.isfile(path):
+            elif os.path.isdir(path) and recursive:
+                files.extend(self._recursive(item))
+            elif os.path.isfile(path) and not recursive:
                 file = self._collect_file_info(path, item)
                 files.append(file)
+            elif os.path.isfile(path) and recursive:
+                logger(logging.WARNING, 'Skipping %s because of --recursive flag' % path)
             else:
                 logger(logging.WARNING, 'No such file or directory: %s' % path)
 
@@ -721,3 +727,69 @@ def _send_trace(self, trace):
         """
         if self.tracing:
             send_trace(trace, self.client.trace_host, self.client.user_agent)
+
+    def _recursive(self, item):
+        """
+        If the --recursive flag is set, it replicates the folder structure recursively into collections
+        A folder only can have either other folders inside or files, but not both of them
+            - If it has folders, the root folder will be a container
+            - If it has files, the root folder will be a dataset
+            - If it is empty, it does not create anything
+
+        :param: item        dictionary containing all descriptions of the files to upload
+        """
+        files = []
+        datasets = []
+        containers = []
+        attach = []
+        scope = item.get('did_scope') if item.get('did_scope') is not None else self.default_file_scope
+        rse = item.get('rse')
+        path = item.get('path')
+        if path[-1] == '/':
+            path = path[0:-1]
+        i = 0
+        path = os.path.abspath(path)
+        for root, dirs, fnames in os.walk(path):
+            if len(dirs) > 0 and len(fnames) > 0 and i == 0:
+                self.logger(logging.ERROR, 'A container can only have either collections or files, not both')
+                raise InputValidationError('Invalid input folder structure')
+            if len(fnames) > 0:
+                datasets.append({'scope': scope, 'name': root.split('/')[-1], 'rse': rse})
+                self.logger(logging.DEBUG, 'Appended dataset with DID %s:%s' % (scope, path))
+                for fname in fnames:
+                    file = self._collect_file_info(os.path.join(root, fname), item)
+                    file['dataset_scope'] = scope
+                    file['dataset_name'] = root.split('/')[-1]
+                    files.append(file)
+                    self.logger(logging.DEBUG, 'Appended file with DID %s:%s' % (scope, fname))
+            elif len(dirs) > 0:
+                containers.append({'scope': scope, 'name': root.split('/')[-1]})
+                self.logger(logging.DEBUG, 'Appended container with DID %s:%s' % (scope, path))
+                attach.extend([{'scope': scope, 'name': root.split('/')[-1], 'rse': rse, 'dids': {'scope': scope, 'name': dir_}} for dir_ in dirs])
+            elif len(dirs) == 0 and len(fnames) == 0:
+                self.logger(logging.WARNING, 'The folder %s is empty, skipping' % root)
+                continue
+            i += 1
+        # if everything went ok, replicate the folder structure in Rucio storage
+        for dataset in datasets:
+            try:
+                self.client.add_dataset(scope=dataset['scope'], name=dataset['name'], rse=dataset['rse'])
+                self.logger(logging.INFO, 'Created dataset with DID %s:%s' % (dataset['scope'], dataset['name']))
+            except RucioException as error:
+                self.logger(logging.ERROR, error)
+                self.logger(logging.ERROR, 'It was not possible to create dataset with DID %s:%s' % (dataset['scope'], dataset['name']))
+        for container in containers:
+            try:
+                self.client.add_container(scope=container['scope'], name=container['name'])
+                self.logger(logging.INFO, 'Created container with DID %s:%s' % (container['scope'], container['name']))
+            except RucioException as error:
+                self.logger(logging.ERROR, error)
+                self.logger(logging.ERROR, 'It was not possible to create dataset with DID %s:%s' % (container['scope'], container['name']))
+        for att in attach:
+            try:
+                self.client.attach_dids(scope=att['scope'], name=att['name'], dids=[att['dids']])
+                self.logger(logging.INFO, 'DIDs attached to collection %s:%s' % (att['scope'], att['name']))
+            except RucioException as error:
+                self.logger(logging.ERROR, error)
+                self.logger(logging.ERROR, 'It was not possible to attach to collection with DID %s:%s' % (att['scope'], att['name']))
+        return files


Test patch:
diff --git a/lib/rucio/tests/temp_factories.py b/lib/rucio/tests/temp_factories.py
--- a/lib/rucio/tests/temp_factories.py
+++ b/lib/rucio/tests/temp_factories.py
@@ -16,7 +16,7 @@
 # Authors:
 # - Radu Carpa <radu.carpa@cern.ch>, 2021
 # - Mayank Sharma <mayank.sharma@cern.ch>, 2021
-
+# - David Población Criado <david.poblacion.criado@cern.ch>, 2021
 
 import os
 import shutil
@@ -323,12 +323,12 @@ def base_dir(self):
 
         return self._base_dir
 
-    def _make_temp_file(self, data, size, namelen, use_basedir):
+    def _make_temp_file(self, data, size, namelen, use_basedir, path):
         fn = ''.join(choice(ascii_uppercase) for x in range(namelen))
         if use_basedir:
-            fp = self.base_dir / fn
+            fp = self.base_dir / path / fn if path is not None else self.base_dir / fn
         else:
-            fp = Path(tempfile.gettempdir(), fn)
+            fp = Path(tempfile.gettempdir()) / path / fn if path is not None else Path(tempfile.gettempdir()) / fn
             self.non_basedir_files.append(fp)
 
         if data is not None:
@@ -343,16 +343,39 @@ def _make_temp_file(self, data, size, namelen, use_basedir):
 
         return fp
 
-    def file_generator(self, data=None, size=2, namelen=10, use_basedir=False):
+    def _make_temp_folder(self, namelen, use_basedir, path):
+        fn = ''.join(choice(ascii_uppercase) for x in range(namelen))
+        if use_basedir:
+            fp = self.base_dir / path / fn if path is not None else self.base_dir / fn
+        else:
+            fp = Path(tempfile.gettempdir()) / path / fn if path is not None else Path(tempfile.gettempdir()) / fn
+            self.non_basedir_files.append(fp)
+
+        os.makedirs(fp, exist_ok=True)
+
+        return fp
+
+    def file_generator(self, data=None, size=2, namelen=10, use_basedir=False, path=None):
         """
         Creates a temporary file
         :param data        : The content to be written in the file. If provided, the size parameter is ignored.
         :param size        : The size of random bytes to be written in the file
         :param namelen     : The length of filename
         :param use_basedir : If True, the file is created under the base_dir for this TemporaryFileFactory instance.
+        :param path        : Relative path of the file, can be under basedir (if use_basedir True) or from the temp dir
         :returns: The absolute path of the generated file
         """
-        return self._make_temp_file(data, size, namelen, use_basedir)
+        return self._make_temp_file(data, size, namelen, use_basedir, path)
+
+    def folder_generator(self, namelen=10, use_basedir=False, path=None):
+        """
+        Creates an empty temporary folder
+        :param namelen     : The length of folder. Only used if path is None.
+        :param use_basedir : If True, the folder is created under the base_dir for this TemporaryFileFactory instance.
+        :param path        : Relative path of the folder, can be under basedir (if use_basedir True).
+        :returns: The absolute path of the generated folder
+        """
+        return self._make_temp_folder(namelen, use_basedir, path)
 
     def cleanup(self):
         for fp in self.non_basedir_files:
diff --git a/lib/rucio/tests/test_bin_rucio.py b/lib/rucio/tests/test_bin_rucio.py
old mode 100644
new mode 100755
--- a/lib/rucio/tests/test_bin_rucio.py
+++ b/lib/rucio/tests/test_bin_rucio.py
@@ -56,6 +56,7 @@
 from rucio.common.utils import generate_uuid, get_tmp_dir, md5, render_json
 from rucio.rse import rsemanager as rsemgr
 from rucio.tests.common import execute, account_name_generator, rse_name_generator, file_generator, scope_name_generator, get_long_vo
+from rucio.tests.temp_factories import TemporaryFileFactory
 
 
 class TestBinRucio(unittest.TestCase):
@@ -1575,3 +1576,79 @@ def test_get_set_delete_limits_rse(self):
         exitcode, out, err = execute(cmd)
         assert re.search("{0}: {1}".format(name3, value3), out) is None
         assert re.search("{0}: {1}".format(name2, value2), out) is not None
+
+    def test_upload_recursive_ok(self):
+        """CLIENT(USER): Upload and preserve folder structure"""
+        with TemporaryFileFactory() as file_factory:
+            folder = file_factory.base_dir
+            folder1 = file_factory.folder_generator(use_basedir=True)
+            folder2 = file_factory.folder_generator(use_basedir=True)
+            folder3 = file_factory.folder_generator(use_basedir=True)
+            folder11 = file_factory.folder_generator(use_basedir=True, path=str(folder1).split('/')[-1])
+            folder12 = file_factory.folder_generator(use_basedir=True, path=str(folder1).split('/')[-1])
+            folder13 = file_factory.folder_generator(use_basedir=True, path=str(folder1).split('/')[-1])
+            file1 = str(file_factory.file_generator(path=folder11)).split('/')[-1]
+            file2 = str(file_factory.file_generator(path=folder2)).split('/')[-1]
+
+            exitcode, out, err = execute('rucio upload --scope %s --rse %s --recursive %s' % (self.user, self.def_rse, folder))
+            print(err)
+            exitcode, out, err = execute('rucio list-content %s:%s' % (self.user, str(folder).split('/')[-1]))
+            assert re.search("{0}:{1}".format(self.user, str(folder1).split('/')[-1]), out) is not None
+            assert re.search("{0}:{1}".format(self.user, str(folder2).split('/')[-1]), out) is not None
+            assert re.search("{0}:{1}".format(self.user, str(folder3).split('/')[-1]), out) is None
+
+            exitcode, out, err = execute('rucio list-content %s:%s' % (self.user, str(folder1).split('/')[-1]))
+            assert re.search("{0}:{1}".format(self.user, str(folder11).split('/')[-1]), out) is not None
+            assert re.search("{0}:{1}".format(self.user, str(folder12).split('/')[-1]), out) is None
+            assert re.search("{0}:{1}".format(self.user, str(folder13).split('/')[-1]), out) is None
+
+            exitcode, out, err = execute('rucio list-content %s:%s' % (self.user, str(folder11).split('/')[-1]))
+            assert re.search("{0}:{1}".format(self.user, file1), out) is not None
+
+            exitcode, out, err = execute('rucio list-content %s:%s' % (self.user, str(folder2).split('/')[-1]))
+            assert re.search("{0}:{1}".format(self.user, file2), out) is not None
+
+    def test_upload_recursive_subfolder(self):
+        """CLIENT(USER): Upload and preserve folder structure in a subfolder"""
+        with TemporaryFileFactory() as file_factory:
+            folder = file_factory.base_dir
+            folder1 = file_factory.folder_generator(use_basedir=True)
+            folder11 = file_factory.folder_generator(use_basedir=True, path=str(folder1).split('/')[-1])
+            file1 = str(file_factory.file_generator(path=folder11)).split('/')[-1]
+
+            execute('rucio upload --scope %s --rse %s --recursive %s' % (self.user, self.def_rse, folder1))
+            exitcode, out, err = execute('rucio list-content %s:%s' % (self.user, str(folder).split('/')[-1]))
+            assert re.search("{0}:{1}".format(self.user, str(folder1).split('/')[-1]), out) is None
+
+            exitcode, out, err = execute('rucio list-content %s:%s' % (self.user, str(folder1).split('/')[-1]))
+            assert re.search("{0}:{1}".format(self.user, str(folder11).split('/')[-1]), out) is not None
+
+            exitcode, out, err = execute('rucio list-content %s:%s' % (self.user, str(folder11).split('/')[-1]))
+            assert re.search("{0}:{1}".format(self.user, file1), out) is not None
+
+    def test_recursive_empty(self):
+        """CLIENT(USER): Upload and preserve folder structure with an empty folder"""
+        with TemporaryFileFactory() as file_factory:
+            folder = file_factory.base_dir
+            folder1 = file_factory.folder_generator(use_basedir=True)
+
+            execute('rucio upload --scope %s --rse %s --recursive %s' % (self.user, self.def_rse, folder))
+            exitcode, out, err = execute('rucio list-content %s:%s' % (self.user, str(folder).split('/')[-1]))
+            assert re.search("{0}:{1}".format(self.user, str(folder1).split('/')[-1]), out) is None
+
+    def test_upload_recursive_only_files(self):
+        """CLIENT(USER): Upload and preserve folder structure only with files"""
+        with TemporaryFileFactory() as file_factory:
+            folder = file_factory.base_dir
+            file1 = str(file_factory.file_generator(use_basedir=True)).split('/')[-1]
+            file2 = str(file_factory.file_generator(use_basedir=True)).split('/')[-1]
+            file3 = str(file_factory.file_generator(use_basedir=True)).split('/')[-1]
+
+            execute('rucio upload --scope %s --rse %s --recursive %s' % (self.user, self.def_rse, folder))
+            exitcode, out, err = execute('rucio ls %s:%s' % (self.user, str(folder).split('/')[-1]))
+            assert re.search("DATASET", out) is not None
+
+            exitcode, out, err = execute('rucio list-content %s:%s' % (self.user, str(folder).split('/')[-1]))
+            assert re.search("{0}:{1}".format(self.user, file1), out) is not None
+            assert re.search("{0}:{1}".format(self.user, file2), out) is not None
+            assert re.search("{0}:{1}".format(self.user, file3), out) is not None