Repository: rucio/rucio

Issue:
Add `ignore_availability` option to the upload client
Motivation
----------

An RSE can get excluded for writing because of lack of free space, as reported by the storage. However, it could be that there’s still some available space, either because of hidden reserves or for small files.

This is important to worker nodes that use the Rucio copytool to stage out their output to a Rucio-managed RSE. If no attempt is made, the CPU time is effectively lost.

Modification
------------

Add an `ignore_availability` parameter and the necessary handling to `UploadClient.upload()`, but do not expose it through the CLI.


Discussion:

Resolving patch:
diff --git a/lib/rucio/client/uploadclient.py b/lib/rucio/client/uploadclient.py
--- a/lib/rucio/client/uploadclient.py
+++ b/lib/rucio/client/uploadclient.py
@@ -39,6 +39,7 @@
 # - Rakshita Varadarajan <rakshitajps@gmail.com>, 2021
 # - Radu Carpa <radu.carpa@cern.ch>, 2021
 # - David Población Criado <david.poblacion.criado@cern.ch>, 2021
+# - Joel Dierkes <joel.dierkes@cern.ch>, 2021
 
 import copy
 import json
@@ -95,7 +96,7 @@ def __init__(self, _client=None, logger=None, tracing=True):
         self.trace['eventType'] = 'upload'
         self.trace['eventVersion'] = version.RUCIO_VERSION[0]
 
-    def upload(self, items, summary_file_path=None, traces_copy_out=None):
+    def upload(self, items, summary_file_path=None, traces_copy_out=None, ignore_availability=False):
         """
         :param items: List of dictionaries. Each dictionary describing a file to upload. Keys:
             path                  - path of the file that will be uploaded
@@ -115,6 +116,7 @@ def upload(self, items, summary_file_path=None, traces_copy_out=None):
             recursive             - Optional: if set, parses the folder structure recursively into collections
         :param summary_file_path: Optional: a path where a summary in form of a json file will be stored
         :param traces_copy_out: reference to an external list, where the traces should be uploaded
+        :param ignore_availability: ignore the availability of a RSE
 
         :returns: 0 on success
 
@@ -147,7 +149,7 @@ def _pick_random_rse(rse_expression):
 
             if not self.rses.get(rse):
                 rse_settings = self.rses.setdefault(rse, rsemgr.get_rse_info(rse, vo=self.client.vo))
-                if rse_settings['availability_write'] != 1:
+                if not ignore_availability and rse_settings['availability_write'] != 1:
                     raise RSEWriteBlocked('%s is not available for writing. No actions have been taken' % rse)
 
             dataset_scope = file.get('dataset_scope')
@@ -219,7 +221,7 @@ def _pick_random_rse(rse_expression):
                 impl = self.preferred_impl(rse_settings, domain)
 
             if not no_register and not register_after_upload:
-                self._register_file(file, registered_dataset_dids)
+                self._register_file(file, registered_dataset_dids, ignore_availability=ignore_availability)
             # if register_after_upload, file should be overwritten if it is not registered
             # otherwise if file already exists on RSE we're done
             if register_after_upload:
@@ -308,7 +310,7 @@ def _pick_random_rse(rse_expression):
 
                 if not no_register:
                     if register_after_upload:
-                        self._register_file(file, registered_dataset_dids)
+                        self._register_file(file, registered_dataset_dids, ignore_availability=ignore_availability)
                     replica_for_api = self._convert_file_for_api(file)
                     if not self.client.update_replicas_states(rse, files=[replica_for_api]):
                         logger(logging.WARNING, 'Failed to update replica state')
@@ -353,7 +355,7 @@ def _pick_random_rse(rse_expression):
             raise NotAllFilesUploaded()
         return 0
 
-    def _register_file(self, file, registered_dataset_dids):
+    def _register_file(self, file, registered_dataset_dids, ignore_availability=False):
         """
         Registers the given file in Rucio. Creates a dataset if
         needed. Registers the file DID and creates the replication
@@ -362,6 +364,7 @@ def _register_file(self, file, registered_dataset_dids):
 
         :param file: dictionary describing the file
         :param registered_dataset_dids: set of dataset dids that were already registered
+        :param ignore_availability: ignore the availability of a RSE
 
         :raises DataIdentifierAlreadyExists: if file DID is already registered and the checksums do not match
         """
@@ -423,7 +426,7 @@ def _register_file(self, file, registered_dataset_dids):
             logger(logging.INFO, 'Successfully added replica in Rucio catalogue at %s' % rse)
             if not dataset_did_str:
                 # only need to add rules for files if no dataset is given
-                self.client.add_replication_rule([file_did], copies=1, rse_expression=rse, lifetime=file.get('lifetime'))
+                self.client.add_replication_rule([file_did], copies=1, rse_expression=rse, lifetime=file.get('lifetime'), ignore_availability=ignore_availability)
                 logger(logging.INFO, 'Successfully added replication rule at %s' % rse)
 
     def _get_file_guid(self, file):


Test patch:
diff --git a/lib/rucio/tests/test_upload.py b/lib/rucio/tests/test_upload.py
--- a/lib/rucio/tests/test_upload.py
+++ b/lib/rucio/tests/test_upload.py
@@ -14,9 +14,10 @@
 # limitations under the License.
 #
 # Authors:
-# - Mayank Sharma <mayank.sharma@cern.ch>, 2021
 # - Radu Carpa <radu.carpa@cern.ch>, 2021
+# - Mayank Sharma <imptodefeat@gmail.com>, 2021
 # - Rakshita Varadarajan <rakshitajps@gmail.com>, 2021
+# - Joel Dierkes <joel.dierkes@cern.ch>, 2021
 
 import json
 import logging
@@ -384,3 +385,19 @@ def test_upload_file_with_supported_protocol_from_config(rse_factory, upload_cli
             mock_put.__name__ = "mock_put"
             upload_client.upload([item])
             mock_put.assert_called()
+
+
+def test_upload_file_ignore_availability(rse_factory, scope, upload_client, file_factory, rucio_client):
+    rse, rse_id = rse_factory.make_posix_rse()
+    rucio_client.update_rse(rse, {'availability_write': False})
+    local_file = file_factory.file_generator()
+    item = [
+        {
+            'path': local_file,
+            'rse': rse,
+            'did_scope': scope,
+        }
+    ]
+
+    status = upload_client.upload(item, ignore_availability=True)
+    assert status == 0