Repository: rucio/rucio

Issue:
Introduce new exclude_site algorithm for chained subscriptions
Motivation
----------
In order to get rid of RPG (https://github.com/rucio/rucio/issues/4508) a new algorithm needs to be introduce for chained subscriptions. This algorithm will exclude the RSEs on the same site as the RSE chosen by the first rule of the chain.




Discussion:

Resolving patch:
diff --git a/lib/rucio/daemons/transmogrifier/transmogrifier.py b/lib/rucio/daemons/transmogrifier/transmogrifier.py
--- a/lib/rucio/daemons/transmogrifier/transmogrifier.py
+++ b/lib/rucio/daemons/transmogrifier/transmogrifier.py
@@ -20,7 +20,7 @@
 import time
 from datetime import datetime
 from json import loads, dumps
-from typing import TYPE_CHECKING, List, Dict, Callable
+from typing import TYPE_CHECKING, List, Dict, Callable, Tuple
 
 import rucio.db.sqla.util
 from rucio.db.sqla.constants import DIDType, SubscriptionState
@@ -122,7 +122,7 @@ def __split_rule_select_rses(
     copies: int,
     blocklisted_rse_id: list,
     logger: "Callable",
-) -> List[Dict]:
+) -> Tuple[List, bool, bool]:
     """
     Internal method to create a list of RSEs that match RSE expression for subscriptions with split_rule.
 
@@ -136,8 +136,10 @@ def __split_rule_select_rses(
     :param copies: The number of copies.
     :param blocklisted_rse_id: The list of blocklisted_rse_id.
     :param logger: The logger.
-    :return: A tuple of list selected_rses, preferred_rses, preferred_unmatched.
+    :return: A tuple with list selected_rses, and 2 booleans create_rule, wont_reevaluate.
     """
+    wont_reevaluate = False
+    create_rule = True
     preferred_rses = set()
     for rule in list_rules(
         filters={
@@ -152,43 +154,59 @@ def __split_rule_select_rses(
         ):
             preferred_rses.add(rse_dict["rse"])
     preferred_rses = list(preferred_rses)
-
-    try:
-        (selected_rses, preferred_unmatched,) = resolve_rse_expression(
-            rse_expression,
-            account,
-            weight=weight,
-            copies=copies,
-            size=0,
-            preferred_rses=preferred_rses,
-            blocklist=blocklisted_rse_id,
-        )
-
-    except (
-        InsufficientTargetRSEs,
-        InsufficientAccountLimit,
-        InvalidRuleWeight,
-        RSEOverQuota,
-    ) as error:
-        logger(
-            logging.WARNING,
-            'Problem getting RSEs for subscription "%s" for account %s : %s. Try including blocklisted sites'
-            % (
-                subscription_name,
+    preferred_unmatched = list()
+    selected_rses = list()
+
+    for attempt in range(0, 2):
+        # First attempt excludes blocklisted RSEs
+        # Second attempt includes blocklisted RSEs
+        try:
+            (selected_rses, preferred_unmatched,) = resolve_rse_expression(
+                rse_expression,
                 account,
-                str(error),
-            ),
-        )
-        # Now including the blocklisted sites
-        (selected_rses, preferred_unmatched,) = resolve_rse_expression(
-            rse_expression,
-            account,
-            weight=weight,
-            copies=copies,
-            size=0,
-            preferred_rses=preferred_rses,
-        )
-    return selected_rses, preferred_rses, preferred_unmatched
+                weight=weight,
+                copies=copies,
+                size=0,
+                preferred_rses=preferred_rses,
+                blocklist=blocklisted_rse_id,
+            )
+            wont_reevaluate = True
+            break
+        except (
+            InsufficientTargetRSEs,
+            InsufficientAccountLimit,
+            InvalidRuleWeight,
+            RSEOverQuota,
+        ) as error:
+            logger(
+                logging.WARNING,
+                'Problem getting RSEs for subscription "%s" for account %s : %s. %s'
+                % (
+                    subscription_name,
+                    account,
+                    str(error),
+                    'Try including blocklisted sites' if attempt == 0 else 'Skipping rule creation.'
+                ),
+            )
+            # Now including the blocklisted sites
+            blocklisted_rse_id = []
+            monitor.record_counter(
+                name="transmogrifier.addnewrule.errortype.{exception}",
+                labels={"exception": str(error.__class__.__name__)},
+            )
+            wont_reevaluate = True
+        except Exception as error:
+            logger(
+                logging.ERROR,
+                "Problem resolving RSE expression %s : %s"
+                % (
+                    rse_expression,
+                    str(error),
+                )
+            )
+    if len(preferred_rses) - len(preferred_unmatched) >= copies:
+        create_rule = False
+    return selected_rses, create_rule, wont_reevaluate
 
 
 def get_subscriptions(logger: "Callable" = logging.log) -> List[Dict]:
@@ -359,7 +377,7 @@ def __is_matching_subscription(subscription, did, metadata):
     return True
 
 
-def select_algorithm(algorithm: str, rule_ids: list, params: dict) -> dict:
+def select_algorithm(algorithm: str, rule_ids: list, params: dict, logger: "Callable") -> dict:
     """
     Method used in case of chained subscriptions
 
@@ -369,15 +387,15 @@ def select_algorithm(algorithm: str, rule_ids: list, params: dict) -> dict:
     :param params: Dictionary of rules parameters to be used by the algorithm
     """
     selected_rses = {}
-    if algorithm == "associated_site":
-        for rule_id in rule_ids:
-            rule = get_rule(rule_id)
-            logging.debug("In select_algorithm, %s", str(rule))
-            rse = rule["rse_expression"]
-            vo = rule["account"].vo
-            if rse_exists(rse, vo=vo):
-                rse_id = get_rse_id(rse, vo=vo)
-                rse_attributes = list_rse_attributes(rse_id)
+    for rule_id in rule_ids:
+        rule = get_rule(rule_id)
+        logging.debug("In select_algorithm, %s", str(rule))
+        rse = rule["rse_expression"]
+        vo = rule["account"].vo
+        if rse_exists(rse, vo=vo):
+            rse_id = get_rse_id(rse, vo=vo)
+            rse_attributes = list_rse_attributes(rse_id)
+            if algorithm == "associated_site":
                 associated_sites = rse_attributes.get("associated_sites", None)
                 associated_site_idx = params.get("associated_site_idx", None)
                 if not associated_site_idx:
@@ -395,14 +413,40 @@ def select_algorithm(algorithm: str, rule_ids: list, params: dict) -> dict:
                         "source_replica_expression": rse,
                         "weight": None,
                     }
-            else:
-                raise SubscriptionWrongParameter(
-                    "Algorithm associated_site only works with split_rule"
-                )
-            if rule["copies"] != 1:
-                raise SubscriptionWrongParameter(
-                    "Algorithm associated_site only works with split_rule"
+            if algorithm == "exclude_site":
+                site = rse_attributes.get("site", None)
+                rse_expression = params['rse_expression'] + '\\site=%s' % site
+                (
+                    selected_rses,
+                    create_rule,
+                    wont_reevaluate,
+                ) = __split_rule_select_rses(
+                    subscription_id=params["subscription_id"],
+                    subscription_name=params["subscription_name"],
+                    scope=rule["scope"],
+                    name=rule["name"],
+                    account=rule.get("account"),
+                    weight=rule.get("weight"),
+                    rse_expression=rse_expression,
+                    copies=rule.get('copies'),
+                    blocklisted_rse_id=params['blocklisted_rse_id'],
+                    logger=logger,
                 )
+                dict_selected_rses = {}
+                for entry in selected_rses:
+                    dict_selected_rses[entry] = {
+                        "source_replica_expression": rse,
+                        "weight": None,
+                    }
+                selected_rses = dict_selected_rses
+        else:
+            raise SubscriptionWrongParameter(
+                "Algorithm %s only works with split_rule" % algorithm
+            )
+        if rule["copies"] != 1:
+            raise SubscriptionWrongParameter(
+                "Algorithm %s only works with split_rule" % algorithm
+            )
     return selected_rses
 
 
@@ -495,6 +539,10 @@ def run_once(heartbeat_handler: "HeartbeatHandler", bulk: int, **_kwargs) -> boo
                     if chained_idx:
                         #  In the case of chained subscription, don't use rseselector but use the rses returned by the algorithm
                         params = {}
+                        params['rse_expression'] = rule_dict.get("rse_expression")
+                        params['subscription_id'] = subscription["id"]
+                        params['subscription_name'] = subscription["name"]
+                        params['blocklisted_rse_id'] = blocklisted_rse_id
                         if rule_dict.get("associated_site_idx", None):
                             params["associated_site_idx"] = rule_dict.get(
                                 "associated_site_idx", None
@@ -506,56 +554,38 @@ def run_once(heartbeat_handler: "HeartbeatHandler", bulk: int, **_kwargs) -> boo
                         )
                         algorithm = rule_dict.get("algorithm", None)
                         selected_rses = select_algorithm(
-                            algorithm, created_rules[chained_idx], params
+                            algorithm,
+                            created_rules[chained_idx],
+                            params,
+                            logger
                         )
                         copies = 1
                     elif split_rule:
-                        try:
-                            (
-                                selected_rses,
-                                preferred_rses,
-                                preferred_unmatched,
-                            ) = __split_rule_select_rses(
-                                subscription_id=subscription["id"],
-                                subscription_name=subscription["name"],
-                                scope=did["scope"],
-                                name=did["name"],
-                                account=rule_dict.get("account"),
-                                weight=weight,
-                                rse_expression=rule_dict.get("rse_expression"),
-                                copies=copies,
-                                blocklisted_rse_id=blocklisted_rse_id,
-                                logger=logger,
-                            )
-                            copies = 1
-                        except (
-                            InsufficientTargetRSEs,
-                            InsufficientAccountLimit,
-                            InvalidRuleWeight,
-                            RSEOverQuota,
-                        ) as error:
-                            logger(
-                                logging.WARNING,
-                                'Problem getting RSEs for subscription "%s" for account %s : %s. Skipping rule creation.'
-                                % (
-                                    subscription["name"],
-                                    rule_dict.get("account"),
-                                    str(error),
-                                ),
-                            )
-                            monitor.record_counter(
-                                name="transmogrifier.addnewrule.errortype.{exception}",
-                                labels={"exception": str(error.__class__.__name__)},
-                            )
-                            # The DID won't be reevaluated at the next cycle
-                            did_success = did_success and True
-                            continue
-
-                        if len(preferred_rses) - len(preferred_unmatched) >= copies:
+                        (
+                            selected_rses,
+                            create_rule,
+                            wont_reevaluate,
+                        ) = __split_rule_select_rses(
+                            subscription_id=subscription["id"],
+                            subscription_name=subscription["name"],
+                            scope=did["scope"],
+                            name=did["name"],
+                            account=rule_dict.get("account"),
+                            weight=weight,
+                            rse_expression=rule_dict.get("rse_expression"),
+                            copies=copies,
+                            blocklisted_rse_id=blocklisted_rse_id,
+                            logger=logger,
+                        )
+                        copies = 1
+                        if not create_rule:
                             continue
+                        # The DID won't be reevaluated at the next cycle
+                        did_success = did_success and wont_reevaluate
 
                     nb_rule = 0
                     #  Try to create the rule
+                    logger(logging.DEBUG, 'selected_rses : %s' % selected_rses)
                     try:
                         for rse in selected_rses:
                             if isinstance(selected_rses, dict):


Test patch:
diff --git a/lib/rucio/tests/test_subscription.py b/lib/rucio/tests/test_subscription.py
--- a/lib/rucio/tests/test_subscription.py
+++ b/lib/rucio/tests/test_subscription.py
@@ -439,8 +439,8 @@ def test_run_transmogrifier_did_type(self, vo, rse_factory, rucio_client):
 
 @pytest.mark.noparallel(reason='uses daemon')
 class TestDaemon:
-    def test_run_transmogrifier_chained_subscription(self, rse_factory, vo, rucio_client, root_account):
-        """ SUBSCRIPTION (DAEMON): Test the transmogrifier with chained subscriptions """
+    def test_run_transmogrifier_chained_subscription_associated_sites_algo(self, rse_factory, vo, rucio_client, root_account):
+        """ SUBSCRIPTION (DAEMON): Test the transmogrifier with chained subscriptions and associated_sites algorithm"""
         activity = get_schema_value('ACTIVITY')['enum'][0]
         rse1, rse1_id = rse_factory.make_mock_rse()
         rse2, rse2_id = rse_factory.make_mock_rse()
@@ -735,3 +735,67 @@ def test_avg_file_size_filtering(self, rse_factory, vo, rucio_client, root_accou
         rule_max_size = [rule for rule in rucio_client.list_did_rules(scope=mock_scope.external, name=dsn[1])]
         assert len(rule_max_size) == 1
         assert rule_max_size[0]['subscription_id'] == str(subid1)
+
+    def test_run_transmogrifier_chained_subscription_exclude_site_algo(self, rse_factory, vo, rucio_client, root_account):
+        """ SUBSCRIPTION (DAEMON): Test the transmogrifier with chained subscriptions and exclude_site algorithm"""
+        activity = get_schema_value('ACTIVITY')['enum'][0]
+        tag_test = uuid()[:8]
+        dict_rse = {}
+        sites = [uuid() for _ in range(3)]
+        for cnt in range(6):
+            rse, rse_id = rse_factory.make_mock_rse()
+            site = sites[cnt // 2]
+            if cnt % 2:
+                rse_type = 'disk'
+            else:
+                rse_type = 'tape'
+            add_rse_attribute(rse_id=rse_id, key='tag', value=tag_test)
+            add_rse_attribute(rse_id=rse_id, key=rse_type, value='True')
+            add_rse_attribute(rse_id=rse_id, key='site', value=site)
+            dict_rse[rse] = {'rse_id': rse_id, 'rse_type': rse_type, 'site': site}
+
+        tmp_scope = InternalScope('mock_' + uuid()[:8], vo=vo)
+        add_scope(tmp_scope, root_account)
+        subscription_name = uuid()
+        dsn_prefix = did_name_generator('dataset')
+
+        rule1 = {'rse_expression': 'tag=%s&disk=True' % tag_test,
+                 'copies': 1,
+                 'activity': activity}
+        rule2 = {'rse_expression': 'tag=%s&tape=True' % tag_test,
+                 'copies': 1,
+                 'activity': activity,
+                 'algorithm': 'exclude_site',
+                 'chained_idx': 1,
+                 'associated_site_idx': 2}
+
+        subid = rucio_client.add_subscription(name=subscription_name,
+                                              account=root_account.external,
+                                              filter_={'scope': [tmp_scope.external, ], 'pattern': '%s.*' % dsn_prefix, 'split_rule': True, 'did_type': ['DATASET', ]},
+                                              replication_rules=[rule1, rule2],
+                                              lifetime=None,
+                                              retroactive=False,
+                                              dry_run=False,
+                                              comments='Test exclude_site algorithm',
+                                              priority=1)
+
+        # Run a few times to check the site is always properly chosen
+        for _ in range(5):
+            dsn = '%sdataset-%s' % (dsn_prefix, uuid())
+            add_did(scope=tmp_scope, name=dsn, did_type=DIDType.DATASET, account=root_account)
+            run(threads=1, bulk=1000000, once=True)
+            rules = [rule for rule in rucio_client.list_did_rules(scope=tmp_scope.external, name=dsn) if str(rule['subscription_id']) == str(subid)]
+            if rules[0]['source_replica_expression']:
+                rules.reverse()
+            print(rules, dict_rse)
+            assert len(rules) == 2
+            chosen_rse = rules[0]['rse_expression']
+            chosen_site = dict_rse[chosen_rse]['site']
+            chosen_rse_type = dict_rse[chosen_rse]['rse_type']
+            assert chosen_rse_type == 'disk'
+
+            chained_rse = rules[1]['rse_expression']
+            chained_site = dict_rse[chained_rse]['site']
+            chained_rse_type = dict_rse[chained_rse]['rse_type']
+            assert chained_rse_type == 'tape'
+            assert chained_site != chosen_site