Repository: rucio/rucio

Issue:
Option to register replica only after upload
Motivation
----------

The current workflow when uploading a file is to register the replica as `UNAVAILABLE` first; Process the upload; Switch the replica to `AVAILABLE`. When the upload fails, the command can just be re-executed, however it is verified that a file with a similar checksum is uploaded.
We recently were told about official workflows where the re-upload, after a failure, would be done with a different file; Thus same filename but different checksum, which does not work in this workflow. The proposed solution is to offer a switch `--register-after-upload` (or something similar) where the client uploads first and registers the replica only as `AVAILABLE` when successful. Thus the pre-registration as `UNAVAILABLE` is skipped. In this case however, if the upload is repeated, the destination file always has to be over-written, as it could always be a new file being uploaded, as long as it is not registered in the catalog yet.


Discussion:

Resolving patch:
diff --git a/lib/rucio/client/uploadclient.py b/lib/rucio/client/uploadclient.py
--- a/lib/rucio/client/uploadclient.py
+++ b/lib/rucio/client/uploadclient.py
@@ -19,6 +19,7 @@
 # - Martin Barisits <martin.barisits@cern.ch>, 2017
 # - Tobias Wegner <twegner@cern.ch>, 2018
 # - Nicolo Magini <nicolo.magini@cern.ch>, 2018
+# - Hannes Hansen <hannes.jakob.hansen@cern.ch>, 2018
 #
 # PY3K COMPATIBLE
 
@@ -71,18 +72,19 @@ def upload(self, items, summary_file_path=None):
         """
 
         :param items: List of dictionaries. Each dictionary describing a file to upload. Keys:
-            path             - path of the file that will be uploaded
-            rse              - rse name (e.g. 'CERN-PROD_DATADISK') where to upload the file
-            did_scope        - Optional: custom did scope (Default: user.<account>)
-            did_name         - Optional: custom did name (Default: name of the file)
-            dataset_scope    - Optional: custom dataset scope
-            dataset_name     - Optional: custom dataset name
-            force_scheme     - Optional: force a specific scheme (if PFN upload this will be overwritten) (Default: None)
-            pfn              - Optional: use a given PFN (this sets no_register to True, and no_register becomes mandatory)
-            no_register      - Optional: if True, the file will not be registered in the rucio catalogue
-            lifetime         - Optional: the lifetime of the file after it was uploaded
-            transfer_timeout - Optional: time after the upload will be aborted
-            guid             - Optional: guid of the file
+            path                  - path of the file that will be uploaded
+            rse                   - rse name (e.g. 'CERN-PROD_DATADISK') where to upload the file
+            did_scope             - Optional: custom did scope (Default: user.<account>)
+            did_name              - Optional: custom did name (Default: name of the file)
+            dataset_scope         - Optional: custom dataset scope
+            dataset_name          - Optional: custom dataset name
+            force_scheme          - Optional: force a specific scheme (if PFN upload this will be overwritten) (Default: None)
+            pfn                   - Optional: use a given PFN (this sets no_register to True, and no_register becomes mandatory)
+            no_register           - Optional: if True, the file will not be registered in the rucio catalogue
+            register_after_upload - Optional: if True, the file will be registered after successful upload
+            lifetime              - Optional: the lifetime of the file after it was uploaded
+            transfer_timeout      - Optional: time after the upload will be aborted
+            guid                  - Optional: guid of the file
         :param summary_file_path: Optional: a path where a summary in form of a json file will be stored
 
         :returns: 0 on success
@@ -131,8 +133,10 @@ def upload(self, items, summary_file_path=None):
             logger.info('Preparing upload for file %s' % basename)
 
             no_register = file.get('no_register')
+            register_after_upload = file.get('register_after_upload') and not no_register
             pfn = file.get('pfn')
             force_scheme = file.get('force_scheme')
+            delete_existing = False
 
             self.trace['scope'] = file['did_scope']
             self.trace['datasetScope'] = file.get('dataset_scope', '')
@@ -153,11 +157,21 @@ def upload(self, items, summary_file_path=None):
                 logger.warning('Upload with given pfn implies that no_register is True, except non-deterministic RSEs')
                 no_register = True
 
-            if not no_register:
+            if not no_register and not register_after_upload:
                 self._register_file(file, registered_dataset_dids)
 
-            # if file already exists on RSE we're done
-            if not is_deterministic and not no_register:
+            # if register_after_upload, file should be overwritten if it is not registered
+            # otherwise if file already exists on RSE we're done
+            if register_after_upload:
+                if rsemgr.exists(rse_settings, file_did):
+                    try:
+                        self.client.get_did(file['did_scope'], file['did_name'])
+                        logger.info('File already registered. Skipping upload.')
+                        continue
+                    except DataIdentifierNotFound:
+                        logger.info('File already exists on RSE. Previous left overs will be overwritten.')
+                        delete_existing = True
+            elif not is_deterministic and not no_register:
                 if rsemgr.exists(rse_settings, pfn):
                     logger.info('File already exists on RSE with given pfn. Skipping upload. Existing replica has to be removed first.')
                     continue
@@ -168,7 +182,6 @@ def upload(self, items, summary_file_path=None):
                 if rsemgr.exists(rse_settings, pfn if pfn else file_did):
                     logger.info('File already exists on RSE. Skipping upload')
                     continue
-
             protocols = rsemgr.get_protocols_ordered(rse_settings=rse_settings, operation='write', scheme=force_scheme)
             protocols.reverse()
             success = False
@@ -193,7 +206,8 @@ def upload(self, items, summary_file_path=None):
                                           source_dir=file['dirname'],
                                           force_scheme=cur_scheme,
                                           force_pfn=pfn,
-                                          transfer_timeout=file.get('transfer_timeout'))
+                                          transfer_timeout=file.get('transfer_timeout'),
+                                          delete_existing=delete_existing)
                     success = state['success']
                     file['upload_result'] = state
                 except (ServiceUnavailable, ResourceTemporaryUnavailable) as error:
@@ -212,6 +226,13 @@ def upload(self, items, summary_file_path=None):
                 if summary_file_path:
                     summary.append(copy.deepcopy(file))
 
+                if not no_register:
+                    if register_after_upload:
+                        self._register_file(file, registered_dataset_dids)
+                    replica_for_api = self._convert_file_for_api(file)
+                    if not self.client.update_replicas_states(rse, files=[replica_for_api]):
+                        logger.warning('Failed to update replica state')
+
                 # add file to dataset if needed
                 if dataset_did_str and not no_register:
                     try:
@@ -219,10 +240,6 @@ def upload(self, items, summary_file_path=None):
                     except Exception as error:
                         logger.warning('Failed to attach file to the dataset')
                         logger.debug(error)
-                if not no_register:
-                    replica_for_api = self._convert_file_for_api(file)
-                    if not self.client.update_replicas_states(rse, files=[replica_for_api]):
-                        logger.warning('Failed to update replica state')
             else:
                 self.trace['clientState'] = 'FAILED'
                 self.trace['stateReason'] = state_reason
diff --git a/lib/rucio/rse/rsemanager.py b/lib/rucio/rse/rsemanager.py
--- a/lib/rucio/rse/rsemanager.py
+++ b/lib/rucio/rse/rsemanager.py
@@ -345,7 +345,7 @@ def exists(rse_settings, files):
     return [gs, ret]
 
 
-def upload(rse_settings, lfns, source_dir=None, force_pfn=None, force_scheme=None, transfer_timeout=None):
+def upload(rse_settings, lfns, source_dir=None, force_pfn=None, force_scheme=None, transfer_timeout=None, delete_existing=False):
     """
         Uploads a file to the connected storage.
         Providing a list indicates the bulk mode.
@@ -402,7 +402,7 @@ def upload(rse_settings, lfns, source_dir=None, force_pfn=None, force_scheme=Non
         if protocol.renaming:
 
             # Check if file replica is already on the storage system
-            if protocol.overwrite is False and protocol.exists(pfn):
+            if protocol.overwrite is False and delete_existing is False and protocol.exists(pfn):
                 ret['%s:%s' % (scope, name)] = exception.FileReplicaAlreadyExists('File %s in scope %s already exists on storage as PFN %s' % (name, scope, pfn))
                 gs = False
             else:
@@ -413,6 +413,16 @@ def upload(rse_settings, lfns, source_dir=None, force_pfn=None, force_scheme=Non
                         ret['%s:%s' % (scope, name)] = exception.RSEOperationNotSupported('Unable to remove temporary file %s.rucio.upload: %s' % (pfn, str(e)))
                         gs = False
                         continue
+
+                if delete_existing:
+                    if protocol.exists('%s' % pfn):  # Check for previous completed uploads that have to be removed before upload
+                        try:
+                            protocol_delete.delete('%s' % list(protocol_delete.lfns2pfns(make_valid_did(lfn)).values())[0])
+                        except Exception as e:
+                            ret['%s:%s' % (scope, name)] = exception.RSEOperationNotSupported('Unable to remove file %s: %s' % (pfn, str(e)))
+                            gs = False
+                            continue
+
                 try:  # Try uploading file
                     protocol.put(base_name, '%s.rucio.upload' % pfn, source_dir, transfer_timeout=transfer_timeout)
                 except Exception as e:
@@ -451,7 +461,7 @@ def upload(rse_settings, lfns, source_dir=None, force_pfn=None, force_scheme=Non
         else:
 
             # Check if file replica is already on the storage system
-            if protocol.overwrite is False and protocol.exists(pfn):
+            if protocol.overwrite is False and delete_existing is False and protocol.exists(pfn):
                 ret['%s:%s' % (scope, name)] = exception.FileReplicaAlreadyExists('File %s in scope %s already exists on storage as PFN %s' % (name, scope, pfn))
                 gs = False
             else:


Test patch:
diff --git a/lib/rucio/tests/test_bin_rucio.py b/lib/rucio/tests/test_bin_rucio.py
--- a/lib/rucio/tests/test_bin_rucio.py
+++ b/lib/rucio/tests/test_bin_rucio.py
@@ -24,6 +24,8 @@
 # - Frank Berghaus <frank.berghaus@cern.ch>, 2017-2018
 # - Tobias Wegner <twegner@cern.ch>, 2018
 # - Hannes Hansen <hannes.jakob.hansen@cern.ch>, 2018
+#
+# PY3K COMPATIBLE
 
 from __future__ import print_function
 
@@ -32,6 +34,7 @@
 import nose.tools
 import re
 
+from rucio.db.sqla import session, models
 from rucio.client.accountlimitclient import AccountLimitClient
 from rucio.client.didclient import DIDClient
 from rucio.client.replicaclient import ReplicaClient
@@ -230,6 +233,52 @@ def test_upload_file(self):
         nose.tools.assert_true((self.upload_success_str % path.basename(tmp_file2)) in out)
         nose.tools.assert_true((self.upload_success_str % path.basename(tmp_file3)) in out)
 
+    def test_upload_file_register_after_upload(self):
+        """CLIENT(USER): Rucio upload files with registration after upload"""
+        # normal upload
+        tmp_file1 = file_generator()
+        tmp_file2 = file_generator()
+        tmp_file3 = file_generator()
+        tmp_file1_name = path.basename(tmp_file1)
+        cmd = 'rucio -v upload --rse {0} --scope {1} {2} {3} {4} --register-after-upload'.format(self.def_rse, self.user, tmp_file1, tmp_file2, tmp_file3)
+        print(self.marker + cmd)
+        exitcode, out, err = execute(cmd)
+        print(out)
+        print(err)
+        remove(tmp_file1)
+        remove(tmp_file2)
+        remove(tmp_file3)
+        nose.tools.assert_true((self.upload_success_str % tmp_file1_name) in out)
+        nose.tools.assert_true((self.upload_success_str % path.basename(tmp_file2)) in out)
+        nose.tools.assert_true((self.upload_success_str % path.basename(tmp_file3)) in out)
+
+        # removing replica -> file on RSE should be overwritten
+        # (simulating an upload error, where a part of the file is uploaded but the replica is not registered)
+        db_session = session.get_session()
+        db_session.query(models.RSEFileAssociation).filter_by(name=tmp_file1_name, scope=self.user).delete()
+        db_session.query(models.ReplicaLock).delete()
+        db_session.query(models.ReplicationRule).filter_by(name=tmp_file1_name, scope=self.user).delete()
+        db_session.query(models.DataIdentifier).filter_by(name=tmp_file1_name, scope=self.user).delete()
+        db_session.commit()
+        tmp_file4 = file_generator()
+        checksum_tmp_file4 = md5(tmp_file4)
+        cmd = 'rucio -v upload --rse {0} --scope {1} --name {2} {3} --register-after-upload'.format(self.def_rse, self.user, tmp_file1_name, tmp_file4)
+        print(self.marker + cmd)
+        exitcode, out, err = execute(cmd)
+        print(out)
+        print(err)
+        nose.tools.assert_true((self.upload_success_str % path.basename(tmp_file4)) in out)
+        nose.tools.assert_equal(checksum_tmp_file4, [replica for replica in self.replica_client.list_replicas(dids=[{'name': tmp_file1_name, 'scope': self.user}])][0]['md5'])
+
+        # try to upload file that already exists on RSE and is already registered -> no overwrite
+        cmd = 'rucio -v upload --rse {0} --scope {1} --name {2} {3} --register-after-upload'.format(self.def_rse, self.user, tmp_file1_name, tmp_file4)
+        print(self.marker + cmd)
+        exitcode, out, err = execute(cmd)
+        print(out)
+        print(err)
+        remove(tmp_file4)
+        nose.tools.assert_true('File already registered' in out)
+
     def test_upload_file_guid(self):
         """CLIENT(USER): Rucio upload file with guid"""
         tmp_file1 = file_generator()
@@ -339,6 +388,30 @@ def test_upload_file_dataset(self):
         print(err)
         nose.tools.assert_not_equal(re.search("{0}:{1}".format(self.user, tmp_file1_name), out), None)
 
+    def test_upload_file_dataset_register_after_upload(self):
+        """CLIENT(USER): Rucio upload files to dataset with file registration after upload"""
+        tmp_file1 = file_generator()
+        tmp_file2 = file_generator()
+        tmp_file3 = file_generator()
+        tmp_file1_name = path.basename(tmp_file1)
+        tmp_dsn = self.user + ':DSet' + rse_name_generator()  # something like mock:DSetMOCK_S0M37HING
+        # Adding files to a new dataset
+        cmd = 'rucio -v upload --register-after-upload --rse {0} --scope {1} {2} {3} {4} {5}'.format(self.def_rse, self.user, tmp_file1, tmp_file2, tmp_file3, tmp_dsn)
+        print(self.marker + cmd)
+        exitcode, out, err = execute(cmd)
+        print(out)
+        print(err)
+        remove(tmp_file1)
+        remove(tmp_file2)
+        remove(tmp_file3)
+        # searching for the file in the new dataset
+        cmd = 'rucio list-files {0}'.format(tmp_dsn)
+        print(self.marker + cmd)
+        exitcode, out, err = execute(cmd)
+        print(out)
+        print(err)
+        nose.tools.assert_not_equal(re.search("{0}:{1}".format(self.user, tmp_file1_name), out), None)
+
     def test_upload_adds_md5digest(self):
         """CLIENT(USER): Upload Checksums"""
         # user has a file to upload