Repository: rucio/rucio

Issue:
rucio get --no-subdir does not check if file already exists
Motivation
----------
From Rod on [JIRA](https://its.cern.ch/jira/browse/RUCIO-2896)

> Running this repeatedly always downloads the file
> rucio get --no-subdir --dir /tmp/walkerr mc15_13TeV:TXT.15373536._052641.tar.gz.1
> DID mc15_13TeV:TXT.15373536._052641.tar.gz.1
> Total files : 1
> Downloaded files : 1
> Files already found locally : 0
> Files that cannot be downloaded : 0
> i.e. Never "Files already found locally : 1"
> Cheers,
> Rod.




Discussion:
For some reason it did state this explicitly: https://github.com/rucio/rucio/blob/36449940cf08788c70e941483630c3b5f81b6a5a/lib/rucio/client/downloadclient.py#L1390
Therefore I don't think this is a bug, but that's just a formality.

Resolving patch:
diff --git a/lib/rucio/client/downloadclient.py b/lib/rucio/client/downloadclient.py
--- a/lib/rucio/client/downloadclient.py
+++ b/lib/rucio/client/downloadclient.py
@@ -220,9 +220,7 @@ def download_pfns(self, items, num_threads=2, trace_custom_fields={}, traces_cop
                 raise InputValidationError('Cannot use PFN download with wildcard in DID')
 
             did_scope, did_name = self._split_did_str(did_str)
-            dest_dir_path = self._prepare_dest_dir(item.get('base_dir', '.'),
-                                                   did_scope, did_name,
-                                                   item.get('no_subdir'))
+            dest_dir_path = self._prepare_dest_dir(item.get('base_dir', '.'), did_scope, item.get('no_subdir'))
 
             item['scope'] = did_scope
             item['name'] = did_name
@@ -1172,7 +1170,7 @@ def _prepare_items_for_download(self, did_to_options, merged_items_with_sources,
 
                     destinations = options['destinations']
                     dataset_scope, dataset_name = self._split_did_str(dataset_did_str)
-                    paths = [os.path.join(self._prepare_dest_dir(dest[0], dataset_name, file_did_name, dest[1]), file_did_name) for dest in destinations]
+                    paths = [os.path.join(self._prepare_dest_dir(dest[0], dataset_name, dest[1]), file_did_name) for dest in destinations]
                     if any(path in all_dest_file_paths for path in paths):
                         raise RucioException("Multiple file items with same destination file path")
 
@@ -1190,7 +1188,7 @@ def _prepare_items_for_download(self, did_to_options, merged_items_with_sources,
                         logger.error('No input options available for %s' % file_did_str)
                         continue
                     destinations = options['destinations']
-                    paths = [os.path.join(self._prepare_dest_dir(dest[0], file_did_scope, file_did_name, dest[1]), file_did_name) for dest in destinations]
+                    paths = [os.path.join(self._prepare_dest_dir(dest[0], file_did_scope, dest[1]), file_did_name) for dest in destinations]
                     if any(path in all_dest_file_paths for path in paths):
                         raise RucioException("Multiple file items with same destination file path")
                     all_dest_file_paths.update(paths)
@@ -1338,7 +1336,7 @@ def _prepare_items_for_download(self, did_to_options, merged_items_with_sources,
                 if pack is None:
                     scope = file_item['scope']
                     first_dest = next(iter(file_item['merged_options']['destinations']))
-                    dest_path = os.path.join(self._prepare_dest_dir(first_dest[0], scope, cea_id, first_dest[1]), cea_id)
+                    dest_path = os.path.join(self._prepare_dest_dir(first_dest[0], scope, first_dest[1]), cea_id)
                     pack = {'scope': scope,
                             'name': cea_id,
                             'dest_file_paths': [dest_path],
@@ -1384,29 +1382,21 @@ def _split_did_str(self, did_str):
 
         return did_scope, did_name
 
-    def _prepare_dest_dir(self, base_dir, dest_dir_name, file_name, no_subdir):
+    @staticmethod
+    def _prepare_dest_dir(base_dir, dest_dir_name, no_subdir):
         """
-        Builds the final destination path for a file and:
-            1. deletes existing files if no_subdir was given
-            2. creates the destination directory if it's not existent
+        Builds the final destination path for a file and creates the
+        destination directory if it's not existent.
         (This function is meant to be used as class internal only)
 
         :param base_dir: base directory part
         :param dest_dir_name: name of the destination directory
-        :param file_name: name of the file that will be downloaded
         :param no_subdir: if no subdirectory should be created
 
         :returns: the absolut path of the destination directory
         """
-        dest_dir_path = os.path.abspath(base_dir)
-        # if no subdirectory is used, existing files will be overwritten
-        if no_subdir:
-            dest_file_path = os.path.join(dest_dir_path, file_name)
-            if os.path.isfile(dest_file_path):
-                self.logger.debug('Deleting existing file: %s' % dest_file_path)
-                os.remove(dest_file_path)
-        else:
-            dest_dir_path = os.path.join(dest_dir_path, dest_dir_name)
+        # append dest_dir_name, if subdir should be used
+        dest_dir_path = os.path.join(os.path.abspath(base_dir), '' if no_subdir else dest_dir_name)
 
         if not os.path.isdir(dest_dir_path):
             os.makedirs(dest_dir_path)


Test patch:
diff --git a/lib/rucio/tests/test_bin_rucio.py b/lib/rucio/tests/test_bin_rucio.py
--- a/lib/rucio/tests/test_bin_rucio.py
+++ b/lib/rucio/tests/test_bin_rucio.py
@@ -581,6 +581,43 @@ def test_download_file(self):
         except Exception:
             pass
 
+    def test_download_no_subdir(self):
+        """CLIENT(USER): Rucio download files with --no-subdir and check that files already found locally are not replaced"""
+        tmp_file = file_generator()
+        # add files
+        cmd = 'rucio upload --rse {0} --scope {1} {2}'.format(self.def_rse, self.user, tmp_file)
+        print(self.marker + cmd)
+        exitcode, out, err = execute(cmd)
+        print(out, err)
+        assert exitcode == 0
+        # download files with --no-subdir
+        cmd = 'rucio -v download --no-subdir --dir /tmp {0}:{1}'.format(self.user, tmp_file[5:])  # triming '/tmp/' from filename
+        print(self.marker + cmd)
+        exitcode, out, err = execute(cmd)
+        print(out, err)
+        assert exitcode == 0
+        # search for the files with ls
+        cmd = 'ls /tmp/'    # search in /tmp/
+        print(self.marker + cmd)
+        exitcode, out, err = execute(cmd)
+        print(out, err)
+        assert tmp_file[5:] in out
+        # download again with --no-subdir
+        cmd = 'rucio -v download --no-subdir --dir /tmp {0}:{1}'.format(self.user, tmp_file[5:])  # triming '/tmp/' from filename
+        print(self.marker + cmd)
+        exitcode, out, err = execute(cmd)
+        print(out, err)
+        assert exitcode == 0
+        assert re.search(r'Downloaded files:\s+0', out) is not None
+        assert re.search(r'Files already found locally:\s+1', out) is not None
+
+        try:
+            for i in listdir('data13_hip'):
+                unlink('data13_hip/%s' % i)
+            rmdir('data13_hip')
+        except Exception:
+            pass
+
     def test_download_filter(self):
         """CLIENT(USER): Rucio download with filter options"""
         # Use filter option to download file with wildcarded name