Repository: rucio/rucio

Issue:
Rule repair does not respect ignore_availability setting
Motivation
----------
If a rule was made with `ignore_availability=True` and a replica lock becomes stuck, any subsequent repair operation should consider `ignore_availability`. Currently it seems that RSE availability is always required before a repair can be submitted, per https://github.com/rucio/rucio/blob/51dd4f09c6541a10186b1e1360e38b6205ea5f28/lib/rucio/core/rule_grouping.py#L770

There may be some question if this is done on purpose for any reason.

Modification
------------
Probably just a change to https://github.com/rucio/rucio/blob/51dd4f09c6541a10186b1e1360e38b6205ea5f28/lib/rucio/core/rule_grouping.py#L770




Discussion:
Any update? I can make a PR with that change though idk about what kind of testing is desired
Hi @nsmith- 
I didn't get to it, if you have time to submit a PR that would be great. I agree that the proposed change should be sufficient, but it has to be done at the three different locations in the repair code (for NONE/ALL/DATASET grouped rules).
For the tests you can probably largely duplicate https://github.com/rucio/rucio/blob/c15716f9494cbbfb1da70e5a77f2d907a61e01a0/lib/rucio/tests/test_judge_repairer.py#L98
But you will need to create a new (test RSE and make it a write-disabled one)

Resolving patch:
diff --git a/lib/rucio/core/rule_grouping.py b/lib/rucio/core/rule_grouping.py
--- a/lib/rucio/core/rule_grouping.py
+++ b/lib/rucio/core/rule_grouping.py
@@ -23,6 +23,7 @@
 # - Andrew Lister <andrew.lister@stfc.ac.uk>, 2019
 # - Luc Goossens <luc.goossens@cern.ch>, 2020
 # - Thomas Beermann <thomas.beermann@cern.ch>, 2021
+# - Nick Smith <nick.smith@cern.ch>, 2021
 
 import logging
 
@@ -576,7 +577,7 @@ def __repair_stuck_locks_with_none_grouping(datasetfiles, locks, replicas, sourc
                 if source_rses:
                     associated_replica = [replica for replica in replicas[(file['scope'], file['name'])] if replica.rse_id == lock.rse_id][0]
                     # Check if there is an eglible source replica for this lock
-                    if set(source_replicas.get((file['scope'], file['name']), [])).intersection(source_rses) and selector_rse_dict.get(lock.rse_id, {}).get('availability_write', True):
+                    if set(source_replicas.get((file['scope'], file['name']), [])).intersection(source_rses) and (selector_rse_dict.get(lock.rse_id, {}).get('availability_write', True) or rule.ignore_availability):
                         __update_lock_replica_and_create_transfer(lock=lock,
                                                                   replica=associated_replica,
                                                                   rule=rule,
@@ -616,7 +617,7 @@ def __repair_stuck_locks_with_none_grouping(datasetfiles, locks, replicas, sourc
                                 locks_to_delete[lock.rse_id] = [lock]
                     except InsufficientTargetRSEs:
                         # Just retry the already existing lock
-                        if __is_retry_required(lock=lock, activity=rule.activity) and selector_rse_dict.get(lock.rse_id, {}).get('availability_write', True):
+                        if __is_retry_required(lock=lock, activity=rule.activity) and (selector_rse_dict.get(lock.rse_id, {}).get('availability_write', True) or rule.ignore_availability):
                             associated_replica = [replica for replica in replicas[(file['scope'], file['name'])] if replica.rse_id == lock.rse_id][0]
                             __update_lock_replica_and_create_transfer(lock=lock,
                                                                       replica=associated_replica,
@@ -683,7 +684,7 @@ def __repair_stuck_locks_with_all_grouping(datasetfiles, locks, replicas, source
                 if source_rses:
                     associated_replica = [replica for replica in replicas[(file['scope'], file['name'])] if replica.rse_id == lock.rse_id][0]
                     # Check if there is an eglible source replica for this lock
-                    if set(source_replicas.get((file['scope'], file['name']), [])).intersection(source_rses) and selector_rse_dict.get(lock.rse_id, {}).get('availability_write', True):
+                    if set(source_replicas.get((file['scope'], file['name']), [])).intersection(source_rses) and (selector_rse_dict.get(lock.rse_id, {}).get('availability_write', True) or rule.ignore_availability):
                         __update_lock_replica_and_create_transfer(lock=lock,
                                                                   replica=associated_replica,
                                                                   rule=rule,
@@ -692,7 +693,7 @@ def __repair_stuck_locks_with_all_grouping(datasetfiles, locks, replicas, source
                                                                   session=session)
                 else:
                     # Just retry the already existing lock
-                    if __is_retry_required(lock=lock, activity=rule.activity) and selector_rse_dict.get(lock.rse_id, {}).get('availability_write', True):
+                    if __is_retry_required(lock=lock, activity=rule.activity) and (selector_rse_dict.get(lock.rse_id, {}).get('availability_write', True) or rule.ignore_availability):
                         associated_replica = [replica for replica in replicas[(file['scope'], file['name'])] if replica.rse_id == lock.rse_id][0]
                         __update_lock_replica_and_create_transfer(lock=lock,
                                                                   replica=associated_replica,
@@ -759,7 +760,7 @@ def __repair_stuck_locks_with_dataset_grouping(datasetfiles, locks, replicas, so
                 if source_rses:
                     associated_replica = [replica for replica in replicas[(file['scope'], file['name'])] if replica.rse_id == lock.rse_id][0]
                     # Check if there is an eglible source replica for this lock
-                    if set(source_replicas.get((file['scope'], file['name']), [])).intersection(source_rses) and selector_rse_dict.get(lock.rse_id, {}).get('availability_write', True):
+                    if set(source_replicas.get((file['scope'], file['name']), [])).intersection(source_rses) and (selector_rse_dict.get(lock.rse_id, {}).get('availability_write', True) or rule.ignore_availability):
                         __update_lock_replica_and_create_transfer(lock=lock,
                                                                   replica=associated_replica,
                                                                   rule=rule,
@@ -768,7 +769,7 @@ def __repair_stuck_locks_with_dataset_grouping(datasetfiles, locks, replicas, so
                                                                   session=session)
                 else:
                     # Just retry the already existing lock
-                    if __is_retry_required(lock=lock, activity=rule.activity) and selector_rse_dict.get(lock.rse_id, {}).get('availability_write', True):
+                    if __is_retry_required(lock=lock, activity=rule.activity) and (selector_rse_dict.get(lock.rse_id, {}).get('availability_write', True) or rule.ignore_availability):
                         associated_replica = [replica for replica in replicas[(file['scope'], file['name'])] if replica.rse_id == lock.rse_id][0]
                         __update_lock_replica_and_create_transfer(lock=lock,
                                                                   replica=associated_replica,


Test patch:
diff --git a/lib/rucio/tests/test_judge_repairer.py b/lib/rucio/tests/test_judge_repairer.py
--- a/lib/rucio/tests/test_judge_repairer.py
+++ b/lib/rucio/tests/test_judge_repairer.py
@@ -23,7 +23,9 @@
 # - Patrick Austin <patrick.austin@stfc.ac.uk>, 2020
 # - Benedikt Ziemons <benedikt.ziemons@cern.ch>, 2020-2021
 # - Simon Fayer <simon.fayer05@imperial.ac.uk>, 2021
+# - Nick Smith <nick.smith@cern.ch>, 2021
 
+import itertools
 import unittest
 from hashlib import sha256
 
@@ -305,29 +307,45 @@ def test_to_repair_a_rule_with_only_1_rse_whose_site_is_blocklisted(self):
 
         rse = rse_name_generator()
         rse_id = add_rse(rse, **self.vo)
-        update_rse(rse_id, {'availability_write': False})
         set_local_account_limit(self.jdoe, rse_id, -1)
-
         rule_repairer(once=True)  # Clean out the repairer
-        scope = InternalScope('mock', **self.vo)
-        files = create_files(4, scope, self.rse4_id, bytes=100)
-        dataset = 'dataset_' + str(uuid())
-        add_did(scope, dataset, DIDType.DATASET, self.jdoe)
-        attach_dids(scope, dataset, files, self.jdoe)
 
-        rule_id = add_rule(dids=[{'scope': scope, 'name': dataset}], account=self.jdoe, copies=1, rse_expression=rse, grouping='DATASET', weight=None, lifetime=None, locked=False, subscription_id=None, ignore_availability=True, activity='DebugJudge')[0]
-
-        assert(RuleState.STUCK == get_rule(rule_id)['state'])
-        rule_repairer(once=True)
-
-        # Stil assert STUCK because of ignore_availability:
-        assert(RuleState.STUCK == get_rule(rule_id)['state'])
-
-        region = make_region().configure('dogpile.cache.memcached',
-                                         expiration_time=3600,
-                                         arguments={'url': config_get('cache', 'url', False, '127.0.0.1:11211'), 'distributed_lock': True})
-        region.delete(sha256(rse.encode()).hexdigest())
-
-        update_rse(rse_id, {'availability_write': True})
-        rule_repairer(once=True)
-        assert(RuleState.REPLICATING == get_rule(rule_id)['state'])
+        region = make_region().configure(
+            'dogpile.cache.memcached',
+            expiration_time=3600,
+            arguments={'url': config_get('cache', 'url', False, '127.0.0.1:11211'), 'distributed_lock': True}
+        )
+
+        def change_availability(new_value):
+            update_rse(rse_id, {'availability_write': new_value})
+            # clear cache
+            region.delete(sha256(rse.encode()).hexdigest())
+
+        for grouping, ignore_availability in itertools.product(["NONE", "DATASET", "ALL"], [True, False]):
+            scope = InternalScope('mock', **self.vo)
+            files = create_files(1, scope, self.rse4_id, bytes=100)
+            dataset = 'dataset_' + str(uuid())
+            add_did(scope, dataset, DIDType.DATASET, self.jdoe)
+            attach_dids(scope, dataset, files, self.jdoe)
+
+            if ignore_availability:
+                change_availability(False)
+                rule_id = add_rule(dids=[{'scope': scope, 'name': dataset}], account=self.jdoe, copies=1, rse_expression=rse, grouping=grouping, weight=None, lifetime=None, locked=False, subscription_id=None, ignore_availability=ignore_availability, activity='DebugJudge')[0]
+                assert(RuleState.STUCK == get_rule(rule_id)['state'])
+
+                rule_repairer(once=True)
+                assert(RuleState.REPLICATING == get_rule(rule_id)['state'])
+
+                change_availability(True)
+            else:
+                rule_id = add_rule(dids=[{'scope': scope, 'name': dataset}], account=self.jdoe, copies=1, rse_expression=rse, grouping=grouping, weight=None, lifetime=None, locked=False, subscription_id=None, ignore_availability=ignore_availability, activity='DebugJudge')[0]
+                failed_transfer(scope=scope, name=files[0]['name'], rse_id=get_replica_locks(scope=files[0]['scope'], name=files[0]['name'])[0].rse_id)
+                change_availability(False)
+                assert(RuleState.STUCK == get_rule(rule_id)['state'])
+
+                rule_repairer(once=True)
+                assert(RuleState.STUCK == get_rule(rule_id)['state'])
+
+                change_availability(True)
+                rule_repairer(once=True)
+                assert(RuleState.REPLICATING == get_rule(rule_id)['state'])