Repository: rucio/rucio

Issue:
Support delayed injection for rules produced by subscriptions
Motivation
----------

One might want the rule created by a subscription to not ‘activate’ until some point in the future.

Modification
------------

Parse and make use of a `delay_injection` parameter.


Discussion:

Resolving patch:
diff --git a/lib/rucio/daemons/transmogrifier/transmogrifier.py b/lib/rucio/daemons/transmogrifier/transmogrifier.py
--- a/lib/rucio/daemons/transmogrifier/transmogrifier.py
+++ b/lib/rucio/daemons/transmogrifier/transmogrifier.py
@@ -19,33 +19,27 @@
 import threading
 import time
 from datetime import datetime
-from json import loads
-from math import exp
-from sys import exc_info
-from traceback import format_exception
-from typing import TYPE_CHECKING
+from json import loads, dumps
+from typing import TYPE_CHECKING, List, Dict, Callable
 
 import rucio.db.sqla.util
 from rucio.db.sqla.constants import DIDType, SubscriptionState
+from rucio.common.config import config_get
 from rucio.common.exception import (
     DatabaseException,
-    DataIdentifierNotFound,
     InvalidReplicationRule,
     DuplicateRule,
-    RSEWriteBlocked,
     InvalidRSEExpression,
     InsufficientTargetRSEs,
     InsufficientAccountLimit,
-    InputValidationError,
     RSEOverQuota,
-    ReplicationRuleCreationTemporaryFailed,
     InvalidRuleWeight,
     StagingAreaRuleRequiresLifetime,
     SubscriptionWrongParameter,
     SubscriptionNotFound,
 )
 from rucio.common.logging import setup_logging
-from rucio.common.schema import validate_schema
+from rucio.common.types import InternalAccount
 from rucio.common.utils import chunks
 from rucio.core import monitor
 from rucio.core.did import list_new_dids, set_new_dids, get_metadata
@@ -58,54 +52,226 @@
 
 if TYPE_CHECKING:
     from rucio.daemons.common import HeartbeatHandler
+    from rucio.common.types import InternalScope
 
 graceful_stop = threading.Event()
 
 RULES_COMMENT_LENGTH = 255
 
 
-def _retrial(func, *args, **kwargs):
+def __get_rule_dict(rule_dict: dict, subscription: dict) -> dict:
     """
-    Retrial method
+    Internal method to clean and enrich the rule_dict coming from the subscription.
+
+    :param rule_dict: The rule dictionnary coming from a subscription.
+    :param subscription: The subscription associated to the rule.
+    :return: A dictionary that contains all the parameters associated to the rule.
     """
-    delay = 0
-    while True:
-        try:
-            return func(*args, **kwargs)
-        except DataIdentifierNotFound as error:
-            logging.warning(error)
-            return 1
-        except DatabaseException as error:
-            logging.error(error)
-            if exp(delay) > 600:
-                logging.error(
-                    "Cannot execute %s after %i attempt. Failing the job."
-                    % (func.__name__, delay)
-                )
-                raise
-            else:
-                logging.error(
-                    "Failure to execute %s. Retrial will be done in %d seconds "
-                    % (func.__name__, exp(delay))
+    source_replica_expression = rule_dict.get("source_replica_expression", None)
+    rule_dict["source_replica_expression"] = source_replica_expression
+    locked = rule_dict.get("locked", None)
+    if locked == "True":
+        locked = True
+    else:
+        locked = False
+    rule_dict["locked"] = locked
+
+    purge_replicas = rule_dict.get("purge_replicas", False)
+    if purge_replicas == "True":
+        purge_replicas = True
+    else:
+        purge_replicas = False
+    rule_dict["purge_replicas"] = purge_replicas
+
+    rule_dict["rse_expression"] = str(rule_dict["rse_expression"])
+    comment = str(subscription["comments"])[:RULES_COMMENT_LENGTH]
+    if "comments" in rule_dict:
+        comment = str(rule_dict["comments"])
+    rule_dict["comment"] = comment
+    account = subscription["account"]
+    if "account" in rule_dict:
+        vo = account.vo
+        account = InternalAccount(rule_dict["account"], vo=vo)
+    rule_dict["account"] = account
+    rule_dict["copies"] = int(rule_dict["copies"])
+    default_activity = config_get("rules", "default_activity", default="default")
+    activity = rule_dict.get("activity", default_activity)
+    rule_dict["activity"] = activity
+    lifetime = rule_dict.get("lifetime", None)
+    if lifetime:
+        rule_dict["lifetime"] = int(lifetime)
+    chained_idx = rule_dict.get("chained_idx", None)
+    if chained_idx:
+        chained_idx = int(rule_dict["copies"])
+    rule_dict["chained_idx"] = chained_idx
+    delay_injection = rule_dict.get("delay_injection", None)
+    if delay_injection:
+        delay_injection = int(delay_injection)
+    rule_dict["delay_injection"] = delay_injection
+    return rule_dict
+
+
+def __split_rule_select_rses(
+    subscription_id: str,
+    subscription_name: str,
+    scope: "InternalScope",
+    name: str,
+    account: "InternalAccount",
+    weight: int,
+    rse_expression: str,
+    copies: int,
+    blocklisted_rse_id: list,
+    logger: "Callable",
+) -> List[Dict]:
+    """
+    Internal method to create a list of RSEs that match RSE expression for subscriptions with split_rule.
+
+    :param subscription_id: The subscription id.
+    :param subscription_name: The subscription name.
+    :param scope: The internal DID scope.
+    :param name: The DID name.
+    :param account: The internal account.
+    :param weight: The weight of the rule.
+    :param rse_expression: The RSE expression of the rule.
+    :param copies: The number of copies.
+    :param blocklisted_rse_id: The list of blocklisted_rse_id.
+    :param logger: The logger.
+    :return: A tuple of list selected_rses, preferred_rses, preferred_unmatched.
+    """
+    preferred_rses = set()
+    for rule in list_rules(
+        filters={
+            "subscription_id": subscription_id,
+            "scope": scope,
+            "name": name,
+        }
+    ):
+        for rse_dict in parse_expression(
+            rule["rse_expression"],
+            filter_={"vo": account.vo},
+        ):
+            preferred_rses.add(rse_dict["rse"])
+    preferred_rses = list(preferred_rses)
+
+    try:
+        (selected_rses, preferred_unmatched,) = resolve_rse_expression(
+            rse_expression,
+            account,
+            weight=weight,
+            copies=copies,
+            size=0,
+            preferred_rses=preferred_rses,
+            blocklist=blocklisted_rse_id,
+        )
+
+    except (
+        InsufficientTargetRSEs,
+        InsufficientAccountLimit,
+        InvalidRuleWeight,
+        RSEOverQuota,
+    ) as error:
+        logger(
+            logging.WARNING,
+            'Problem getting RSEs for subscription "%s" for account %s : %s. Try including blocklisted sites'
+            % (
+                subscription_name,
+                account,
+                str(error),
+            ),
+        )
+        # Now including the blocklisted sites
+        (selected_rses, preferred_unmatched,) = resolve_rse_expression(
+            rse_expression,
+            account,
+            weight=weight,
+            copies=copies,
+            size=0,
+            preferred_rses=preferred_rses,
+        )
+    return selected_rses, preferred_rses, preferred_unmatched
+
+
+def get_subscriptions(logger: "Callable" = logging.log) -> List[Dict]:
+    """
+    A method to extract the list of active subscriptions and exclued the one that have bad RSE expression.
+    :param logger: The logger.
+    :return: The list of active subscriptions.
+    """
+    subscriptions = []
+    try:
+        sub_dict = {3: []}
+        #  Get the list of subscriptions. The default priority of the subscription is 3. 0 is the highest priority, 5 the lowest
+        #  The priority is defined as 'policyid'
+        logger(logging.DEBUG, "Listing active subscriptions")
+        for sub in list_subscriptions(None, None):
+            rse_expression = sub.get("rse_expression")
+            skip_sub = False
+            rules = loads(sub["replication_rules"])
+            overwrite_rules = False
+            for rule in rules:
+                rse_expression = rule.get("rse_expression")
+                try:
+                    list_rses_from_expression = parse_expression(rse_expression)
+                except InvalidRSEExpression:
+                    logger(
+                        logging.ERROR,
+                        "Invalid RSE expression %s for subscription %s. Subscription removed from the list",
+                        rse_expression,
+                        sub["id"],
+                    )
+                    skip_sub = True
+                    break
+                if rule.get("copies") == "*":
+                    rule["copies"] = len(list_rses_from_expression)
+                    overwrite_rules = True
+            if skip_sub:
+                continue
+            if overwrite_rules:
+                sub["replication_rules"] = dumps(rules)
+            if (
+                sub["state"] != SubscriptionState.INACTIVE
+                and sub["lifetime"]
+                and (datetime.now() > sub["lifetime"])
+            ):
+                update_subscription(
+                    name=sub["name"],
+                    account=sub["account"],
+                    metadata={"state": SubscriptionState.INACTIVE},
                 )
-            time.sleep(exp(delay))
-            delay += 1
-        except Exception:
-            exc_type, exc_value, exc_traceback = exc_info()
-            logging.critical(
-                "".join(format_exception(exc_type, exc_value, exc_traceback)).strip()
-            )
-            raise
+
+            elif sub["state"] in [SubscriptionState.ACTIVE, SubscriptionState.UPDATED]:
+                priority = 3
+                if "policyid" in sub:
+                    if int(sub["policyid"]) not in sub_dict:
+                        sub_dict[int(sub["policyid"])] = []
+                    priority = int(sub["policyid"])
+                sub_dict[priority].append(sub)
+        priorities = list(sub_dict.keys())
+        priorities.sort()
+        #  Order the subscriptions according to their priority
+        for priority in priorities:
+            subscriptions.extend(sub_dict[priority])
+        logger(logging.INFO, "%i active subscriptions", len(subscriptions))
+    except SubscriptionNotFound as error:
+        logger(logging.WARNING, "No subscriptions defined: %s" % (str(error)))
+        return []
+    except Exception as error:
+        logger(
+            logging.ERROR,
+            "Failed to get list of new DIDs or subscriptions: %s" % (str(error)),
+        )
+        return []
+    return subscriptions
 
 
-def is_matching_subscription(subscription, did, metadata):
+def __is_matching_subscription(subscription, did, metadata):
     """
-    Method to identify if a DID matches a subscription.
+    Internal method to identify if a DID matches a subscription.
 
-    param subscription: The subscription dictionnary.
-    param did: The DID dictionnary
-    param metadata: The metadata dictionnary for the DID
-    return: True/False
+    :param subscription: The subscription dictionary.
+    :param did: The DID dictionary
+    :param metadata: The metadata dictionnary for the DID
+    :return: True/False
     """
     if metadata["hidden"]:
         return False
@@ -174,14 +340,14 @@ def is_matching_subscription(subscription, did, metadata):
     return True
 
 
-def select_algorithm(algorithm, rule_ids, params):
+def select_algorithm(algorithm: str, rule_ids: list, params: dict) -> dict:
     """
     Method used in case of chained subscriptions
 
     :param algorithm: Algorithm used for the chained rule. Now only associated_site
                       associated_site : Choose an associated endpoint according to the RSE attribute assoiciated_site
     :param rule_ids: List of parent rules
-    :param params: List of rules parameters to be used by the algorithm
+    :param params: Dictionary of rules parameters to be used by the algorithm
     """
     selected_rses = {}
     if algorithm == "associated_site":
@@ -248,405 +414,225 @@ def transmogrifier(bulk: int = 5, once: bool = False, sleep_time: int = 60) -> N
 def run_once(heartbeat_handler: "HeartbeatHandler", bulk: int, **_kwargs) -> bool:
 
     worker_number, total_workers, logger = heartbeat_handler.live()
-    dids, subscriptions = [], []
-    try:
-        #  Get the new DIDs based on the is_new flag
-        logger(logging.DEBUG, "Listing new dids")
-        for did in list_new_dids(
-            thread=worker_number,
-            total_threads=total_workers,
-            chunk_size=bulk,
-            did_type=None,
-        ):
-            dids.append(
-                {
-                    "scope": did["scope"],
-                    "did_type": str(did["did_type"]),
-                    "name": did["name"],
-                }
-            )
-        logger(logging.INFO, "%i new dids to process", len(dids))
-
-        sub_dict = {3: []}
-        #  Get the list of subscriptions. The default priority of the subscription is 3. 0 is the highest priority, 5 the lowest
-        #  The priority is defined as 'policyid'
-        logger(logging.DEBUG, "Listing active subscriptions")
-        for sub in list_subscriptions(None, None):
-            if (
-                sub["state"] != SubscriptionState.INACTIVE
-                and sub["lifetime"]
-                and (datetime.now() > sub["lifetime"])
-            ):
-                update_subscription(
-                    name=sub["name"],
-                    account=sub["account"],
-                    metadata={"state": SubscriptionState.INACTIVE},
-                )
-
-            elif sub["state"] in [SubscriptionState.ACTIVE, SubscriptionState.UPDATED]:
-                priority = 3
-                if "policyid" in sub:
-                    if int(sub["policyid"]) not in sub_dict:
-                        sub_dict[int(sub["policyid"])] = []
-                    priority = int(sub["policyid"])
-                sub_dict[priority].append(sub)
-        priorities = list(sub_dict.keys())
-        priorities.sort()
-        #  Order the subscriptions according to their priority
-        for priority in priorities:
-            subscriptions.extend(sub_dict[priority])
-        logger(logging.INFO, "%i active subscriptions", len(subscriptions))
-    except SubscriptionNotFound as error:
-        logger(logging.WARNING, "No subscriptions defined: %s" % (str(error)))
-        must_sleep = True
-        return must_sleep
-    except Exception as error:
-        logger(
-            logging.ERROR,
-            "Failed to get list of new DIDs or subscriptions: %s" % (str(error)),
-        )
-        must_sleep = False
-        return must_sleep
-
-    results = {}
     timer = monitor.Timer()
     blocklisted_rse_id = [rse["id"] for rse in list_rses({"availability_write": False})]
     identifiers = []
+    #  List all the active subscriptions
+    subscriptions = get_subscriptions(logger=logger)
+
     #  Loop over all the new dids
-    for did in dids:
+    #  Get the new DIDs based on the is_new flag
+    logger(logging.DEBUG, "Listing new dids")
+    for did in list_new_dids(
+        thread=worker_number,
+        total_threads=total_workers,
+        chunk_size=bulk,
+        did_type=None,
+    ):
         _, _, logger = heartbeat_handler.live()
         did_success = True
-        if did["did_type"] == str(DIDType.DATASET) or did["did_type"] == str(
-            DIDType.CONTAINER
+        if not (
+            did["did_type"] == DIDType.DATASET or did["did_type"] == DIDType.CONTAINER
         ):
-            did_tag = "%s:%s" % (did["scope"].internal, did["name"])
-            results[did_tag] = []
-            try:
-                metadata = get_metadata(did["scope"], did["name"])
-                # Loop over all the subscriptions
-                for subscription in subscriptions:
-                    #  Check if the DID match the subscription
-                    if is_matching_subscription(subscription, did, metadata) is True:
-                        filter_string = loads(subscription["filter"])
-                        split_rule = filter_string.get("split_rule", False)
-                        stime = time.time()
-                        results[did_tag].append(subscription["id"])
+            identifiers.append(
+                {
+                    "scope": did["scope"],
+                    "name": did["name"],
+                    "did_type": did["did_type"],
+                }
+            )
+            continue
+        metadata = get_metadata(did["scope"], did["name"])
+
+        #  Loop over all the subscriptions
+        for subscription in subscriptions:
+            #  Check if the DID match the subscription
+            if __is_matching_subscription(subscription, did, metadata) is True:
+                filter_string = loads(subscription["filter"])
+                split_rule = filter_string.get("split_rule", False)
+                stime = time.time()
+                logger(
+                    logging.INFO,
+                    "%s:%s matches subscription %s"
+                    % (did["scope"], did["name"], subscription["name"]),
+                )
+                rules = loads(subscription["replication_rules"])
+                created_rules = {}
+                for cnt, rule_dict in enumerate(rules):
+                    created_rules[cnt + 1] = []
+                    #  Get all the rule and subscription parameters
+                    rule_dict = __get_rule_dict(rule_dict, subscription)
+                    weight = rule_dict.get("weight", None)
+                    source_replica_expression = rule_dict.get(
+                        "source_replica_expression", None
+                    )
+                    copies = rule_dict["copies"]
+                    success = False
+
+                    chained_idx = rule_dict.get("chained_idx", None)
+                    #  By default selected_rses contains only the rse_expression
+                    #  It is overwritten in 2 cases : Chained subscription and split_rule
+                    selected_rses = [rule_dict.get("rse_expression")]
+                    if chained_idx:
+                        #  In the case of chained subscription, don't use rseselector but use the rses returned by the algorithm
+                        params = {}
+                        if rule_dict.get("associated_site_idx", None):
+                            params["associated_site_idx"] = rule_dict.get(
+                                "associated_site_idx", None
+                            )
                         logger(
-                            logging.INFO,
-                            "%s:%s matches subscription %s"
-                            % (did["scope"], did["name"], subscription["name"]),
+                            logging.DEBUG,
+                            "Chained subscription identified. Will use %s",
+                            str(created_rules[chained_idx]),
+                        )
+                        algorithm = rule_dict.get("algorithm", None)
+                        selected_rses = select_algorithm(
+                            algorithm, created_rules[chained_idx], params
                         )
-                        rules = loads(subscription["replication_rules"])
-                        created_rules = {}
-                        cnt = 0
-                        for rule_dict in rules:
-                            cnt += 1
-                            created_rules[cnt] = []
-                            # Get all the rule and subscription parameters
-                            grouping = rule_dict.get("grouping", "DATASET")
-                            lifetime = rule_dict.get("lifetime", None)
-                            ignore_availability = rule_dict.get(
-                                "ignore_availability", None
+                        copies = 1
+                    elif split_rule:
+                        try:
+                            (
+                                selected_rses,
+                                preferred_rses,
+                                preferred_unmatched,
+                            ) = __split_rule_select_rses(
+                                subscription_id=subscription["id"],
+                                subscription_name=subscription["name"],
+                                scope=did["scope"],
+                                name=did["name"],
+                                account=rule_dict.get("account"),
+                                weight=weight,
+                                rse_expression=rule_dict.get("rse_expression"),
+                                copies=copies,
+                                blocklisted_rse_id=blocklisted_rse_id,
+                                logger=logger,
                             )
-                            weight = rule_dict.get("weight", None)
-                            source_replica_expression = rule_dict.get(
-                                "source_replica_expression", None
+                            copies = 1
+                        except (
+                            InsufficientTargetRSEs,
+                            InsufficientAccountLimit,
+                            InvalidRuleWeight,
+                            RSEOverQuota,
+                        ) as error:
+                            logger(
+                                logging.WARNING,
+                                'Problem getting RSEs for subscription "%s" for account %s : %s. Skipping rule creation.'
+                                % (
+                                    subscription["name"],
+                                    rule_dict.get("account"),
+                                    str(error),
+                                ),
                             )
-                            locked = rule_dict.get("locked", None)
-                            if locked == "True":
-                                locked = True
-                            else:
-                                locked = False
-                            purge_replicas = rule_dict.get("purge_replicas", False)
-                            if purge_replicas == "True":
-                                purge_replicas = True
-                            else:
-                                purge_replicas = False
-                            rse_expression = str(rule_dict["rse_expression"])
-                            comment = str(subscription["comments"])[
-                                :RULES_COMMENT_LENGTH
-                            ]
-                            if "comments" in rule_dict:
-                                comment = str(rule_dict["comments"])
-                            subscription_id = str(subscription["id"])
-                            account = subscription["account"]
-                            copies = int(rule_dict["copies"])
-                            activity = rule_dict.get("activity", "User Subscriptions")
-                            try:
-                                validate_schema(
-                                    name="activity", obj=activity, vo=account.vo
-                                )
-                            except InputValidationError as error:
-                                logger(
-                                    logging.ERROR,
-                                    "Error validating the activity %s" % (str(error)),
-                                )
-                                activity = "User Subscriptions"
-                            if lifetime:
-                                lifetime = int(lifetime)
-
-                            str_activity = "".join(activity.split())
-                            success = False
-                            nattempt = 5
-                            attemptnr = 0
-                            skip_rule_creation = False
-
-                            selected_rses = []
-                            chained_idx = rule_dict.get("chained_idx", None)
-                            if chained_idx:
-                                params = {}
-                                if rule_dict.get("associated_site_idx", None):
-                                    params["associated_site_idx"] = rule_dict.get(
-                                        "associated_site_idx", None
-                                    )
-                                logger(
-                                    logging.DEBUG,
-                                    "Chained subscription identified. Will use %s",
-                                    str(created_rules[chained_idx]),
-                                )
-                                algorithm = rule_dict.get("algorithm", None)
-                                selected_rses = select_algorithm(
-                                    algorithm, created_rules[chained_idx], params
-                                )
-                            else:
-                                # In the case of chained subscription, don't use rseselector but use the rses returned by the algorithm
-                                if split_rule:
-                                    preferred_rses = set()
-                                    for rule in list_rules(
-                                        filters={
-                                            "subscription_id": subscription_id,
-                                            "scope": did["scope"],
-                                            "name": did["name"],
-                                        }
-                                    ):
-                                        for rse_dict in parse_expression(
-                                            rule["rse_expression"],
-                                            filter_={"vo": account.vo},
-                                        ):
-                                            preferred_rses.add(rse_dict["rse"])
-                                    preferred_rses = list(preferred_rses)
-
-                                    try:
-                                        (
-                                            selected_rses,
-                                            preferred_unmatched,
-                                        ) = resolve_rse_expression(
-                                            rse_expression,
-                                            account,
-                                            weight=weight,
-                                            copies=copies,
-                                            size=0,
-                                            preferred_rses=preferred_rses,
-                                            blocklist=blocklisted_rse_id,
-                                        )
-
-                                    except (
-                                        InsufficientTargetRSEs,
-                                        InsufficientAccountLimit,
-                                        InvalidRuleWeight,
-                                        RSEOverQuota,
-                                    ) as error:
-                                        logger(
-                                            logging.WARNING,
-                                            'Problem getting RSEs for subscription "%s" for account %s : %s. Try including blocklisted sites'
-                                            % (
-                                                subscription["name"],
-                                                account,
-                                                str(error),
-                                            ),
-                                        )
-                                        # Now including the blocklisted sites
-                                        try:
-                                            (
-                                                selected_rses,
-                                                preferred_unmatched,
-                                            ) = resolve_rse_expression(
-                                                rse_expression,
-                                                account,
-                                                weight=weight,
-                                                copies=copies,
-                                                size=0,
-                                                preferred_rses=preferred_rses,
-                                            )
-                                            ignore_availability = True
-                                        except (
-                                            InsufficientTargetRSEs,
-                                            InsufficientAccountLimit,
-                                            InvalidRuleWeight,
-                                            RSEOverQuota,
-                                        ) as error:
-                                            logger(
-                                                logging.ERROR,
-                                                'Problem getting RSEs for subscription "%s" for account %s : %s. Skipping rule creation.'
-                                                % (
-                                                    subscription["name"],
-                                                    account,
-                                                    str(error),
-                                                ),
-                                            )
-                                            monitor.record_counter(
-                                                name="transmogrifier.addnewrule.errortype.{exception}",
-                                                labels={
-                                                    "exception": str(
-                                                        error.__class__.__name__
-                                                    )
-                                                },
-                                            )
-                                            # The DID won't be reevaluated at the next cycle
-                                            did_success = did_success and True
-                                            continue
-
-                                    if (
-                                        len(preferred_rses) - len(preferred_unmatched)
-                                        >= copies
-                                    ):
-                                        skip_rule_creation = True
-
-                            for attempt in range(0, nattempt):
-                                attemptnr = attempt
-                                nb_rule = 0
-                                #  Try to create the rule
-                                try:
-                                    if split_rule:
-                                        if not skip_rule_creation:
-                                            for rse in selected_rses:
-                                                if isinstance(selected_rses, dict):
-                                                    source_replica_expression = (
-                                                        selected_rses[rse].get(
-                                                            "source_replica_expression",
-                                                            None,
-                                                        )
-                                                    )
-                                                    weight = selected_rses[rse].get(
-                                                        "weight", None
-                                                    )
-                                                logger(
-                                                    logging.INFO,
-                                                    "Will insert one rule for %s:%s on %s"
-                                                    % (did["scope"], did["name"], rse),
-                                                )
-                                                rule_ids = add_rule(
-                                                    dids=[
-                                                        {
-                                                            "scope": did["scope"],
-                                                            "name": did["name"],
-                                                        }
-                                                    ],
-                                                    account=account,
-                                                    copies=1,
-                                                    rse_expression=rse,
-                                                    grouping=grouping,
-                                                    weight=weight,
-                                                    lifetime=lifetime,
-                                                    locked=locked,
-                                                    subscription_id=subscription_id,
-                                                    source_replica_expression=source_replica_expression,
-                                                    activity=activity,
-                                                    purge_replicas=purge_replicas,
-                                                    ignore_availability=ignore_availability,
-                                                    comment=comment,
-                                                )
-                                                created_rules[cnt].append(rule_ids[0])
-                                                nb_rule += 1
-                                                if nb_rule == copies:
-                                                    success = True
-                                                    break
-                                    else:
-                                        rule_ids = add_rule(
-                                            dids=[
-                                                {
-                                                    "scope": did["scope"],
-                                                    "name": did["name"],
-                                                }
-                                            ],
-                                            account=account,
-                                            copies=copies,
-                                            rse_expression=rse_expression,
-                                            grouping=grouping,
-                                            weight=weight,
-                                            lifetime=lifetime,
-                                            locked=locked,
-                                            subscription_id=subscription["id"],
-                                            source_replica_expression=source_replica_expression,
-                                            activity=activity,
-                                            purge_replicas=purge_replicas,
-                                            ignore_availability=ignore_availability,
-                                            comment=comment,
-                                        )
-                                        created_rules[cnt].append(rule_ids[0])
-                                        nb_rule += 1
-                                    monitor.record_counter(
-                                        name="transmogrifier.addnewrule.done",
-                                        delta=nb_rule,
-                                    )
-                                    monitor.record_counter(
-                                        name="transmogrifier.addnewrule.activity.{activity}",
-                                        delta=nb_rule,
-                                        labels={"activity": str_activity},
-                                    )
-                                    success = True
-                                    break
-                                except (
-                                    InvalidReplicationRule,
-                                    InvalidRuleWeight,
-                                    InvalidRSEExpression,
-                                    StagingAreaRuleRequiresLifetime,
-                                    DuplicateRule,
-                                ) as error:
-                                    # Errors that won't be retried
-                                    success = True
-                                    logger(logging.ERROR, str(error))
-                                    monitor.record_counter(
-                                        name="transmogrifier.addnewrule.errortype.{exception}",
-                                        labels={
-                                            "exception": str(error.__class__.__name__)
-                                        },
-                                    )
-                                    break
-                                except (
-                                    ReplicationRuleCreationTemporaryFailed,
-                                    InsufficientTargetRSEs,
-                                    InsufficientAccountLimit,
-                                    DatabaseException,
-                                    RSEWriteBlocked,
-                                ) as error:
-                                    # Errors to be retried
-                                    logger(
-                                        logging.ERROR,
-                                        "%s Will perform an other attempt %i/%i"
-                                        % (str(error), attempt + 1, nattempt),
-                                    )
-                                    monitor.record_counter(
-                                        name="transmogrifier.addnewrule.errortype.{exception}",
-                                        labels={
-                                            "exception": str(error.__class__.__name__)
-                                        },
-                                    )
-                                except Exception:
-                                    # Unexpected errors
-                                    monitor.record_counter(
-                                        name="transmogrifier.addnewrule.errortype.{exception}",
-                                        labels={"exception": "unknown"},
-                                    )
-                                    logger(
-                                        logging.ERROR, "Unexpected error", exc_info=True
-                                    )
-
-                            did_success = did_success and success
-                            if (attemptnr + 1) == nattempt and not success:
-                                logger(
-                                    logging.ERROR,
-                                    "Rule for %s:%s on %s cannot be inserted"
-                                    % (did["scope"], did["name"], rse_expression),
-                                )
-                            else:
-                                logger(
-                                    logging.INFO,
-                                    "%s rule(s) inserted in %f seconds"
-                                    % (str(nb_rule), time.time() - stime),
+                            monitor.record_counter(
+                                name="transmogrifier.addnewrule.errortype.{exception}",
+                                labels={"exception": str(error.__class__.__name__)},
+                            )
+                            # The DID won't be reevaluated at the next cycle
+                            did_success = did_success and True
+                            continue
+
+                        if len(preferred_rses) - len(preferred_unmatched) >= copies:
+                            continue
+
+                    nb_rule = 0
+                    #  Try to create the rule
+                    try:
+                        for rse in selected_rses:
+                            if isinstance(selected_rses, dict):
+                                #  selected_rses is a dictionary only when split_rule is True or for chained subscriptions
+                                source_replica_expression = selected_rses[rse].get(
+                                    "source_replica_expression",
+                                    None,
                                 )
-            except DataIdentifierNotFound as error:
-                logger(logging.WARNING, str(error))
+                                weight = selected_rses[rse].get("weight", None)
+                            logger(
+                                logging.INFO,
+                                "Will insert one rule for %s:%s on %s"
+                                % (did["scope"], did["name"], rse),
+                            )
+                            rule_ids = add_rule(
+                                dids=[
+                                    {
+                                        "scope": did["scope"],
+                                        "name": did["name"],
+                                    }
+                                ],
+                                account=rule_dict.get("account"),
+                                copies=copies,
+                                rse_expression=rse,
+                                grouping=rule_dict.get("grouping", "DATASET"),
+                                weight=weight,
+                                lifetime=rule_dict.get("lifetime", None),
+                                locked=rule_dict.get("locked", None),
+                                subscription_id=subscription["id"],
+                                source_replica_expression=source_replica_expression,
+                                activity=rule_dict.get("activity"),
+                                purge_replicas=rule_dict.get("purge_replicas", False),
+                                ignore_availability=rule_dict.get(
+                                    "ignore_availability", None
+                                ),
+                                comment=rule_dict.get("comment"),
+                                delay_injection=rule_dict.get("delay_injection"),
+                            )
+                            created_rules[cnt + 1].append(rule_ids[0])
+                            nb_rule += 1
+                            if nb_rule == copies:
+                                success = True
+                            if split_rule:
+                                success = True
+
+                        monitor.record_counter(
+                            name="transmogrifier.addnewrule.done",
+                            delta=nb_rule,
+                        )
+                        monitor.record_counter(
+                            name="transmogrifier.addnewrule.activity.{activity}",
+                            delta=nb_rule,
+                            labels={
+                                "activity": "".join(rule_dict.get("activity").split())
+                            },
+                        )
+                        success = True
+                    except (
+                        InvalidReplicationRule,
+                        InvalidRuleWeight,
+                        InvalidRSEExpression,
+                        StagingAreaRuleRequiresLifetime,
+                        DuplicateRule,
+                    ) as error:
+                        # Errors that won't be retried
+                        success = True
+                        logger(logging.ERROR, str(error))
+                        monitor.record_counter(
+                            name="transmogrifier.addnewrule.errortype.{exception}",
+                            labels={"exception": str(error.__class__.__name__)},
+                        )
+                    except Exception:
+                        # Errors that will be retried
+                        monitor.record_counter(
+                            name="transmogrifier.addnewrule.errortype.{exception}",
+                            labels={"exception": "unknown"},
+                        )
+                        logger(logging.ERROR, "Unexpected error", exc_info=True)
+
+                    did_success = did_success and success
+                    if not success:
+                        logger(
+                            logging.ERROR,
+                            "Rule for %s:%s on %s cannot be inserted"
+                            % (
+                                did["scope"],
+                                did["name"],
+                                rule_dict.get("rse_expression"),
+                            ),
+                        )
+                    else:
+                        logger(
+                            logging.INFO,
+                            "%s rule(s) inserted in %f seconds"
+                            % (str(nb_rule), time.time() - stime),
+                        )
 
         if did_success:
             if did["did_type"] == str(DIDType.FILE):
@@ -669,7 +655,7 @@ def run_once(heartbeat_handler: "HeartbeatHandler", bulk: int, **_kwargs) -> boo
     #  Mark the DIDs as processed
     flag_timer = monitor.Timer()
     for identifier in chunks(identifiers, 100):
-        _retrial(set_new_dids, identifier, None)
+        set_new_dids(identifier, None)
     logger(logging.DEBUG, "Time to set the new flag : %f" % flag_timer.elapsed)
 
     timer.stop()
@@ -680,8 +666,11 @@ def run_once(heartbeat_handler: "HeartbeatHandler", bulk: int, **_kwargs) -> boo
             account=sub["account"],
             metadata={"last_processed": datetime.now()},
         )
-    logger(logging.INFO, "It took %f seconds to process %i DIDs" % (timer.elapsed, len(dids)))
-    logger(logging.DEBUG, "DIDs processed : %s" % (str(dids)))
+    logger(
+        logging.INFO,
+        "It took %f seconds to process %i DIDs" % (timer.elapsed, len(identifiers)),
+    )
+    logger(logging.DEBUG, "DIDs processed : %s" % (str(identifiers)))
     monitor.record_counter(name="transmogrifier.job.done", delta=1)
     timer.record("transmogrifier.job.duration")
     must_sleep = True


Test patch:
diff --git a/lib/rucio/tests/test_rse_expression_parser.py b/lib/rucio/tests/test_rse_expression_parser.py
--- a/lib/rucio/tests/test_rse_expression_parser.py
+++ b/lib/rucio/tests/test_rse_expression_parser.py
@@ -50,6 +50,8 @@ def setUp(self):
             self.vo = {}
             self.filter = {'filter_': {'vo': 'def'}}
 
+        self.already_existing_rses = rse.list_rses()
+
         self.rse1 = rse_name_generator()
         self.rse2 = rse_name_generator()
         self.rse3 = rse_name_generator()
@@ -117,8 +119,16 @@ def test_attribute_reference(self):
     def test_all_rse(self):
         """ RSE_EXPRESSION_PARSER (CORE) Test reference on all RSE """
         all_rses = rse.list_rses(filters=self.filter['filter_'])
-        value = sorted(rse_expression_parser.parse_expression("*", **self.filter), key=lambda rse: rse['rse'])
-        expected = sorted(all_rses, key=lambda rse: rse['rse'])
+        rse_expression_parser.REGION.invalidate()
+        value = rse_expression_parser.parse_expression("*", **self.filter)
+        for rse_ in self.already_existing_rses:
+            if rse_ in all_rses:
+                all_rses.remove(rse_)
+            if rse_ in value:
+                value.remove(rse_)
+        value = sorted(value, key=lambda rse_: rse_['rse'])
+        expected = sorted(all_rses, key=lambda rse_: rse_['rse'])
+        assert len(value) == 5
         assert value == expected
 
     def test_tag_reference(self):
diff --git a/lib/rucio/tests/test_subscription.py b/lib/rucio/tests/test_subscription.py
--- a/lib/rucio/tests/test_subscription.py
+++ b/lib/rucio/tests/test_subscription.py
@@ -14,12 +14,14 @@
 # limitations under the License.
 
 import unittest
+from datetime import datetime
 from json import loads
 
 import pytest
 
 from rucio.api.subscription import list_subscriptions, add_subscription, update_subscription, \
     list_subscription_rule_states, get_subscription_by_id
+from rucio.db.sqla.constants import RuleState
 from rucio.client.didclient import DIDClient
 from rucio.client.subscriptionclient import SubscriptionClient
 from rucio.common.config import config_get_bool
@@ -28,12 +30,13 @@
 from rucio.common.types import InternalAccount, InternalScope
 from rucio.common.utils import generate_uuid as uuid
 from rucio.core.account import add_account
-from rucio.core.did import add_did, set_new_dids, list_new_dids
+from rucio.core.did import add_did, set_new_dids, list_new_dids, attach_dids
 from rucio.core.rule import add_rule
+from rucio.core.rse import add_rse_attribute
 from rucio.core.scope import add_scope
-from rucio.daemons.transmogrifier.transmogrifier import run
+from rucio.daemons.transmogrifier.transmogrifier import run, get_subscriptions
 from rucio.db.sqla.constants import AccountType, DIDType
-from rucio.tests.common import headers, auth, did_name_generator
+from rucio.tests.common import headers, auth, did_name_generator, rse_name_generator
 from rucio.tests.common_server import get_vo
 
 
@@ -395,7 +398,7 @@ def test_create_and_list_subscription_by_name(self):
         result = [sub['id'] for sub in self.sub_client.list_subscriptions(name=subscription_name)]
         assert subid == result[0]
 
-    @pytest.mark.noparallel(reason='runs transfmogrifier. Cannot be run at the same time with other tests running it')
+    @pytest.mark.noparallel(reason='runs transmogrifier. Cannot be run at the same time with other tests running it')
     def test_run_transmogrifier(self):
         """ SUBSCRIPTION (DAEMON): Test the transmogrifier and the split_rule mode """
         new_dids = [did for did in list_new_dids(did_type=None, thread=None, total_threads=None, chunk_size=100000, session=None)]
@@ -424,7 +427,7 @@ def test_run_transmogrifier(self):
         rules = [rule for rule in self.did_client.list_did_rules(scope=tmp_scope.external, name=dsn) if str(rule['subscription_id']) == str(subid)]
         assert len(rules) == 2
 
-    @pytest.mark.noparallel(reason='runs transfmogrifier. Cannot be run at the same time with other tests running it')
+    @pytest.mark.noparallel(reason='runs transmogrifier. Cannot be run at the same time with other tests running it')
     def test_run_transmogrifier_did_type(self):
         """ SUBSCRIPTION (DAEMON): Test the transmogrifier with did_type subscriptions """
         new_dids = [did for did in list_new_dids(did_type=None, thread=None, total_threads=None, chunk_size=100000, session=None)]
@@ -451,3 +454,191 @@ def test_run_transmogrifier_did_type(self):
         run(threads=1, bulk=1000000, once=True)
         rules = [rule for rule in self.did_client.list_did_rules(scope=tmp_scope.external, name=dsn) if str(rule['subscription_id']) == str(subid)]
         assert len(rules) == 2
+
+
+@pytest.mark.noparallel(reason='uses daemon')
+class TestDaemon():
+    def test_run_transmogrifier_chained_subscription(self, rse_factory, vo, rucio_client, root_account):
+        """ SUBSCRIPTION (DAEMON): Test the transmogrifier with chained subscriptions """
+        activity = get_schema_value('ACTIVITY')['enum'][0]
+        rse1, rse1_id = rse_factory.make_mock_rse()
+        rse2, rse2_id = rse_factory.make_mock_rse()
+        rse3, _ = rse_factory.make_mock_rse()
+        rse4, _ = rse_factory.make_mock_rse()
+        rse5, _ = rse_factory.make_mock_rse()
+        rse6, _ = rse_factory.make_mock_rse()
+        add_rse_attribute(rse_id=rse1_id, key='associated_sites', value='%s,%s' % (rse3, rse4))
+        add_rse_attribute(rse_id=rse2_id, key='associated_sites', value='%s,%s' % (rse5, rse6))
+        rses = []
+        for cnt in range(5):
+            rse, _ = rse_factory.make_mock_rse()
+            rses.append(rse)
+        rse_expression = '%s|%s' % (rse1, rse2)
+        tmp_scope = InternalScope('mock_' + uuid()[:8], vo=vo)
+        add_scope(tmp_scope, root_account)
+        subscription_name = uuid()
+        dsn_prefix = did_name_generator('dataset')
+        dsn = '%sdataset-%s' % (dsn_prefix, uuid())
+
+        add_did(scope=tmp_scope, name=dsn, did_type=DIDType.DATASET, account=root_account)
+        rule1 = {'rse_expression': rse_expression,
+                 'copies': 1,
+                 'activity': activity}
+        rule2 = {'rse_expression': '*',
+                 'copies': 1,
+                 'activity': activity,
+                 'algorithm': 'associated_site',
+                 'chained_idx': 1,
+                 'associated_site_idx': 2}
+
+        subid = rucio_client.add_subscription(name=subscription_name,
+                                              account=root_account.external,
+                                              filter_={'scope': [tmp_scope.external, ], 'pattern': '%s.*' % dsn_prefix, 'split_rule': True, 'did_type': ['DATASET', ]},
+                                              replication_rules=[rule1, rule2],
+                                              lifetime=None,
+                                              retroactive=0,
+                                              dry_run=0,
+                                              comments='Ni ! Ni!',
+                                              priority=1)
+        run(threads=1, bulk=1000000, once=True)
+        rules = [rule for rule in rucio_client.list_did_rules(scope=tmp_scope.external, name=dsn) if str(rule['subscription_id']) == str(subid)]
+        assert len(rules) == 2
+        if rules[0]['source_replica_expression']:
+            rules.reverse()
+        assert rules[0]['rse_expression'] in [rse1, rse2]
+        if rules[0]['rse_expression'] == rse1:
+            assert rules[1]['source_replica_expression'] == rse1
+            assert rules[1]['rse_expression'] == rse4
+        if rules[0]['rse_expression'] == rse2:
+            assert rules[1]['source_replica_expression'] == rse2
+            assert rules[1]['rse_expression'] == rse6
+
+    def test_skip_subscription_bad_rse_expression(self, rse_factory, vo, rucio_client, root_account):
+        """ SUBSCRIPTION (DAEMON): Check that the subscriptions with bad RSE expression are skipped"""
+        activity = get_schema_value('ACTIVITY')['enum'][0]
+        _, _ = rse_factory.make_mock_rse()
+        rse_expression = rse_name_generator()
+        tmp_scope = InternalScope('mock_' + uuid()[:8], vo=vo)
+        add_scope(tmp_scope, root_account)
+        subscription_name = uuid()
+        dsn_prefix = did_name_generator('dataset')
+        dsn = '%sdataset-%s' % (dsn_prefix, uuid())
+
+        add_did(scope=tmp_scope, name=dsn, did_type=DIDType.DATASET, account=root_account)
+        rule = {'rse_expression': rse_expression,
+                'copies': 1,
+                'activity': activity,
+                'rse_expression': rse_expression}
+
+        rucio_client.add_subscription(name=subscription_name,
+                                      account=root_account.external,
+                                      filter_={'scope': [tmp_scope.external, ], 'pattern': '%s.*' % dsn_prefix, 'split_rule': True, 'did_type': ['DATASET', ]},
+                                      replication_rules=[rule],
+                                      lifetime=None,
+                                      retroactive=0,
+                                      dry_run=0,
+                                      comments='Ni ! Ni!',
+                                      priority=1)
+        for sub in get_subscriptions():
+            for rule in loads(sub["replication_rules"]):
+                assert rule["rse_expression"] != rse_expression
+
+    def test_run_transmogrifier_wildcard_copies(self, rse_factory, vo, rucio_client, root_account):
+        """ SUBSCRIPTION (DAEMON): Test the transmogrifier with wildcard copies """
+        activity = get_schema_value('ACTIVITY')['enum'][0]
+        rse_attribute = uuid()[:8]
+        rses = {'no_tag': [], rse_attribute: []}
+        for cnt in range(5):
+            rse, rse_id = rse_factory.make_mock_rse()
+            rses['no_tag'].append(rse)
+        for cnt in range(5):
+            rse, rse_id = rse_factory.make_mock_rse()
+            add_rse_attribute(rse_id=rse_id, key=rse_attribute, value=True)
+            rses[rse_attribute].append(rse)
+        rse_expression = rse_attribute
+        tmp_scope = InternalScope('mock_' + uuid()[:8], vo=vo)
+        add_scope(tmp_scope, root_account)
+
+        # Check without split rule
+        subscription_name = uuid()
+        dsn_prefix = did_name_generator('dataset')
+        dsn = '%sdataset-%s' % (dsn_prefix, uuid())
+
+        add_did(scope=tmp_scope, name=dsn, did_type=DIDType.DATASET, account=root_account)
+        rule = {'rse_expression': rse_expression,
+                'copies': '*',
+                'activity': activity,
+                'rse_expression': rse_expression}
+
+        subid = rucio_client.add_subscription(name=subscription_name,
+                                              account=root_account.external,
+                                              filter_={'scope': [tmp_scope.external, ], 'pattern': '%s.*' % dsn_prefix, 'split_rule': True, 'did_type': ['DATASET', ]},
+                                              replication_rules=[rule],
+                                              lifetime=None,
+                                              retroactive=0,
+                                              dry_run=0,
+                                              comments='Ni ! Ni!',
+                                              priority=1)
+        run(threads=1, bulk=1000000, once=True)
+        rules = [rule for rule in rucio_client.list_did_rules(scope=tmp_scope.external, name=dsn) if str(rule['subscription_id']) == str(subid)]
+
+        # Check with split rule
+        subscription_name = uuid()
+        dsn_prefix = did_name_generator('dataset')
+        dsn = '%sdataset-%s' % (dsn_prefix, uuid())
+
+        add_did(scope=tmp_scope, name=dsn, did_type=DIDType.DATASET, account=root_account)
+        rule = {'rse_expression': rse_expression,
+                'copies': '*',
+                'activity': activity,
+                'rse_expression': rse_expression}
+
+        subid = rucio_client.add_subscription(name=subscription_name,
+                                              account=root_account.external,
+                                              filter_={'scope': [tmp_scope.external, ], 'pattern': '%s.*' % dsn_prefix, 'split_rule': False, 'did_type': ['DATASET', ]},
+                                              replication_rules=[rule],
+                                              lifetime=None,
+                                              retroactive=0,
+                                              dry_run=0,
+                                              comments='Ni ! Ni!',
+                                              priority=1)
+        run(threads=1, bulk=1000000, once=True)
+        rules = [rule for rule in rucio_client.list_did_rules(scope=tmp_scope.external, name=dsn) if str(rule['subscription_id']) == str(subid)]
+        assert len(rules) == 1
+        assert rules[0]['copies'] == 5
+
+    def test_run_transmogrifier_delayed_subscription(self, rse_factory, vo, rucio_client, root_account, mock_scope):
+        """ SUBSCRIPTION (DAEMON): Test the transmogrifier with delayed subscription """
+        activity = get_schema_value('ACTIVITY')['enum'][0]
+        nbfiles = 3
+        rse1, _ = rse_factory.make_mock_rse()
+        rse2, rse2_id = rse_factory.make_mock_rse()
+        rse3, _ = rse_factory.make_mock_rse()
+        rse_expression = rse1
+        subscription_name = uuid()
+        dsn_prefix = did_name_generator('dataset')
+        dsn = '%sdataset-%s' % (dsn_prefix, uuid())
+
+        files = [{'scope': mock_scope, 'name': did_name_generator('file'), 'bytes': 1, 'adler32': '0cc737eb', 'meta': {'events': 10}} for _ in range(nbfiles)]
+        add_did(scope=mock_scope, name=dsn, did_type=DIDType.DATASET, account=root_account)
+        attach_dids(scope=mock_scope, name=dsn, rse_id=rse2_id, dids=files, account=root_account)
+        rule = {'rse_expression': rse_expression,
+                'copies': 1,
+                'activity': activity,
+                'delay_injection': 86500}
+
+        subid = rucio_client.add_subscription(name=subscription_name,
+                                              account=root_account.external,
+                                              filter_={'scope': [mock_scope.external, ], 'pattern': '%s.*' % dsn_prefix, 'split_rule': True, 'did_type': ['DATASET', ]},
+                                              replication_rules=[rule],
+                                              lifetime=None,
+                                              retroactive=0,
+                                              dry_run=0,
+                                              comments='Ni ! Ni!',
+                                              priority=1)
+        run(threads=1, bulk=1000000, once=True)
+        rules = [rule for rule in rucio_client.list_did_rules(scope=mock_scope.external, name=dsn) if str(rule['subscription_id']) == str(subid)]
+        print(rules)
+        assert rules[0]['rse_expression'] == rse_expression
+        assert rules[0]['state'] == RuleState.INJECT.name
+        assert (rules[0]['created_at'] - datetime.now()).days == 1