Repository: rucio/rucio

Issue:
Possibility to have chained subscriptions
Motivation
----------
Chained subscription use case for Belle II :
- RAW data produce at T0
- 1st subscription exports the RAW to one DATADISK at site A
- 2nd subscription exports the RAW from DATADISK at site A to DATATAPE at the same site






Discussion:

Resolving patch:
diff --git a/lib/rucio/common/exception.py b/lib/rucio/common/exception.py
--- a/lib/rucio/common/exception.py
+++ b/lib/rucio/common/exception.py
@@ -19,7 +19,7 @@
 # - Mario Lassnig <mario.lassnig@cern.ch>, 2012-2017
 # - Vincent Garonne <vgaronne@gmail.com>, 2012-2018
 # - Ralph Vigne <ralph.vigne@cern.ch>, 2012-2013
-# - Cedric Serfon <cedric.serfon@cern.ch>, 2013-2019
+# - Cedric Serfon <cedric.serfon@cern.ch>, 2013-2020
 # - Wen Guan <wguan.icedew@gmail.com>, 2014-2015
 # - Tobias Wegner <twegner@cern.ch>, 2018
 # - Hannes Hansen <hannes.jakob.hansen@cern.ch>, 2018-2019
@@ -972,3 +972,33 @@ def __init__(self, *args, **kwargs):
         super(NoDistance, self).__init__(*args, **kwargs)
         self._message = 'Cannot found a distance between 2 RSEs'
         self.error_code = 92
+
+
+class PolicyPackageNotFound(RucioException):
+    """
+    The policy package specified in the config file cannot be loaded.
+    """
+    def __init__(self, *args, **kwargs):
+        super(PolicyPackageNotFound, self).__init__(*args, **kwargs)
+        self._message = 'The specified policy package cannot be loaded'
+        self.error_code = 93
+
+
+class CannotAuthorize(RucioException):
+    """
+    Failed to authorize an operation.
+    """
+    def __init__(self, *args, **kwargs):
+        super(CannotAuthorize, self).__init__(*args, **kwargs)
+        self._message = 'Can not authorize operation.'
+        self.error_code = 94
+
+
+class SubscriptionWrongParameter(RucioException):
+    """
+    RucioException
+    """
+    def __init__(self, *args, **kwargs):
+        super(SubscriptionWrongParameter, self).__init__(*args, **kwargs)
+        self._message = "Subscription wrong parameters"
+        self.error_code = 95
diff --git a/lib/rucio/daemons/transmogrifier/transmogrifier.py b/lib/rucio/daemons/transmogrifier/transmogrifier.py
--- a/lib/rucio/daemons/transmogrifier/transmogrifier.py
+++ b/lib/rucio/daemons/transmogrifier/transmogrifier.py
@@ -13,7 +13,7 @@
 # limitations under the License.
 #
 # Authors:
-# - Cedric Serfon <cedric.serfon@cern.ch>, 2013-2018
+# - Cedric Serfon <cedric.serfon@cern.ch>, 2013-2020
 # - Vincent Garonne <vgaronne@gmail.com>, 2014-2018
 # - David Cameron <d.g.cameron@gmail.com>, 2014
 # - Mario Lassnig <mario.lassnig@cern.ch>, 2015
@@ -42,16 +42,17 @@
 from rucio.db.sqla.constants import DIDType, SubscriptionState
 from rucio.common.exception import (DatabaseException, DataIdentifierNotFound, InvalidReplicationRule, DuplicateRule, RSEBlacklisted,
                                     InvalidRSEExpression, InsufficientTargetRSEs, InsufficientAccountLimit, InputValidationError, RSEOverQuota,
-                                    ReplicationRuleCreationTemporaryFailed, InvalidRuleWeight, StagingAreaRuleRequiresLifetime, SubscriptionNotFound)
+                                    ReplicationRuleCreationTemporaryFailed, InvalidRuleWeight, StagingAreaRuleRequiresLifetime,
+                                    SubscriptionWrongParameter, SubscriptionNotFound)
 from rucio.common.config import config_get
 from rucio.common.schema import validate_schema
 from rucio.common.utils import chunks
 from rucio.core import monitor, heartbeat
 from rucio.core.did import list_new_dids, set_new_dids, get_metadata
-from rucio.core.rse import list_rses
+from rucio.core.rse import list_rses, rse_exists, get_rse_id, list_rse_attributes
 from rucio.core.rse_expression_parser import parse_expression
 from rucio.core.rse_selector import RSESelector
-from rucio.core.rule import add_rule, list_rules
+from rucio.core.rule import add_rule, list_rules, get_rule
 from rucio.core.subscription import list_subscriptions, update_subscription
 
 
@@ -146,6 +147,41 @@ def is_matching_subscription(subscription, did, metadata):
     return True
 
 
+def select_algorithm(algorithm, rule_ids, params):
+    """
+    Method used in case of chained subscriptions
+
+    :param algorithm: Algorithm used for the chained rule. Now only associated_site
+                      associated_site : Choose an associated endpoint according to the RSE attribute assoiciated_site
+    :param rule_ids: List of parent rules
+    :param params: List of rules parameters to be used by the algorithm
+    """
+    selected_rses = {}
+    if algorithm == 'associated_site':
+        for rule_id in rule_ids:
+            rule = get_rule(rule_id)
+            logging.debug('In select_algorithm, %s', str(rule))
+            rse = rule['rse_expression']
+            if rse_exists(rse):
+                rse_id = get_rse_id(rse)
+                rse_attributes = list_rse_attributes(rse_id)
+                associated_sites = rse_attributes.get('associated_sites', None)
+                associated_site_idx = params.get('associated_site_idx', None)
+                if not associated_site_idx:
+                    raise SubscriptionWrongParameter('Missing parameter associated_site_idx')
+                if associated_sites:
+                    associated_sites = associated_sites.split(',')
+                    if associated_site_idx > len(associated_sites) + 1:
+                        raise SubscriptionWrongParameter('Parameter associated_site_idx is out of range')
+                    associated_site = associated_sites[associated_site_idx - 1]
+                    selected_rses[associated_site] = {'source_replica_expression': rse, 'weight': None}
+            else:
+                raise SubscriptionWrongParameter('Algorithm associated_site only works with split_rule')
+            if rule['copies'] != 1:
+                raise SubscriptionWrongParameter('Algorithm associated_site only works with split_rule')
+    return selected_rses
+
+
 def transmogrifier(bulk=5, once=False, sleep_time=60):
     """
     Creates a Transmogrifier Worker that gets a list of new DIDs for a given hash,
@@ -223,36 +259,37 @@ def transmogrifier(bulk=5, once=False, sleep_time=60):
                             if is_matching_subscription(subscription, did, metadata) is True:
                                 filter_string = loads(subscription['filter'])
                                 split_rule = filter_string.get('split_rule', False)
-                                if split_rule == 'true':
-                                    split_rule = True
-                                elif split_rule == 'false':
-                                    split_rule = False
                                 stime = time.time()
                                 results[did_tag].append(subscription['id'])
                                 logging.info(prepend_str + '%s:%s matches subscription %s' % (did['scope'], did['name'], subscription['name']))
-                                for rule_string in loads(subscription['replication_rules']):
+                                rules = loads(subscription['replication_rules'])
+                                created_rules = {}
+                                cnt = 0
+                                for rule_dict in rules:
+                                    cnt += 1
+                                    created_rules[cnt] = []
                                     # Get all the rule and subscription parameters
-                                    grouping = rule_string.get('grouping', 'DATASET')
-                                    lifetime = rule_string.get('lifetime', None)
-                                    ignore_availability = rule_string.get('ignore_availability', None)
-                                    weight = rule_string.get('weight', None)
-                                    source_replica_expression = rule_string.get('source_replica_expression', None)
-                                    locked = rule_string.get('locked', None)
+                                    grouping = rule_dict.get('grouping', 'DATASET')
+                                    lifetime = rule_dict.get('lifetime', None)
+                                    ignore_availability = rule_dict.get('ignore_availability', None)
+                                    weight = rule_dict.get('weight', None)
+                                    source_replica_expression = rule_dict.get('source_replica_expression', None)
+                                    locked = rule_dict.get('locked', None)
                                     if locked == 'True':
                                         locked = True
                                     else:
                                         locked = False
-                                    purge_replicas = rule_string.get('purge_replicas', False)
+                                    purge_replicas = rule_dict.get('purge_replicas', False)
                                     if purge_replicas == 'True':
                                         purge_replicas = True
                                     else:
                                         purge_replicas = False
-                                    rse_expression = str(rule_string['rse_expression'])
+                                    rse_expression = str(rule_dict['rse_expression'])
                                     comment = str(subscription['comments'])
                                     subscription_id = str(subscription['id'])
                                     account = subscription['account']
-                                    copies = int(rule_string['copies'])
-                                    activity = rule_string.get('activity', 'User Subscriptions')
+                                    copies = int(rule_dict['copies'])
+                                    activity = rule_dict.get('activity', 'User Subscriptions')
                                     try:
                                         validate_schema(name='activity', obj=activity)
                                     except InputValidationError as error:
@@ -267,40 +304,50 @@ def transmogrifier(bulk=5, once=False, sleep_time=60):
                                     attemptnr = 0
                                     skip_rule_creation = False
 
-                                    if split_rule:
-                                        rses = parse_expression(rse_expression)
-                                        list_of_rses = [rse['id'] for rse in rses]
-                                        # Check that some rule doesn't already exist for this DID and subscription
-                                        preferred_rse_ids = []
-                                        for rule in list_rules(filters={'subscription_id': subscription_id, 'scope': did['scope'], 'name': did['name']}):
-                                            already_existing_rses = [(rse['rse'], rse['id']) for rse in parse_expression(rule['rse_expression'])]
-                                            for rse, rse_id in already_existing_rses:
-                                                if (rse_id in list_of_rses) and (rse_id not in preferred_rse_ids):
-                                                    preferred_rse_ids.append(rse_id)
-                                        if len(preferred_rse_ids) >= copies:
-                                            skip_rule_creation = True
-
-                                        rse_id_dict = {}
-                                        for rse in rses:
-                                            rse_id_dict[rse['id']] = rse['rse']
-                                        try:
-                                            rseselector = RSESelector(account=account, rses=rses, weight=weight, copies=copies - len(preferred_rse_ids))
-                                            selected_rses = [rse_id_dict[rse_id] for rse_id, _, _ in rseselector.select_rse(0, preferred_rse_ids=preferred_rse_ids, copies=copies, blacklist=blacklisted_rse_id)]
-                                        except (InsufficientTargetRSEs, InsufficientAccountLimit, InvalidRuleWeight, RSEOverQuota) as error:
-                                            logging.warning(prepend_str + 'Problem getting RSEs for subscription "%s" for account %s : %s. Try including blacklisted sites' %
-                                                            (subscription['name'], account, str(error)))
-                                            # Now including the blacklisted sites
+                                    selected_rses = []
+                                    chained_idx = rule_dict.get('chained_idx', None)
+                                    if chained_idx:
+                                        params = {}
+                                        if rule_dict.get('associated_site_idx', None):
+                                            params['associated_site_idx'] = rule_dict.get('associated_site_idx', None)
+                                        logging.debug('%s Chained subscription identified. Will use %s', prepend_str, str(created_rules[chained_idx]))
+                                        algorithm = rule_dict.get('algorithm', None)
+                                        selected_rses = select_algorithm(algorithm, created_rules[chained_idx], params)
+                                    else:
+                                        # In the case of chained subscription, don't use rseselector but use the rses returned by the algorithm
+                                        if split_rule:
+                                            rses = parse_expression(rse_expression)
+                                            list_of_rses = [rse['id'] for rse in rses]
+                                            # Check that some rule doesn't already exist for this DID and subscription
+                                            preferred_rse_ids = []
+                                            for rule in list_rules(filters={'subscription_id': subscription_id, 'scope': did['scope'], 'name': did['name']}):
+                                                already_existing_rses = [(rse['rse'], rse['id']) for rse in parse_expression(rule['rse_expression'])]
+                                                for rse, rse_id in already_existing_rses:
+                                                    if (rse_id in list_of_rses) and (rse_id not in preferred_rse_ids):
+                                                        preferred_rse_ids.append(rse_id)
+                                            if len(preferred_rse_ids) >= copies:
+                                                skip_rule_creation = True
+                                            rse_id_dict = {}
+                                            for rse in rses:
+                                                rse_id_dict[rse['id']] = rse['rse']
                                             try:
                                                 rseselector = RSESelector(account=account, rses=rses, weight=weight, copies=copies - len(preferred_rse_ids))
-                                                selected_rses = [rse_id_dict[rse_id] for rse_id, _, _ in rseselector.select_rse(0, preferred_rse_ids=preferred_rse_ids, copies=copies, blacklist=[])]
-                                                ignore_availability = True
+                                                selected_rses = [rse_id_dict[rse_id] for rse_id, _, _ in rseselector.select_rse(0, preferred_rse_ids=preferred_rse_ids, copies=copies, blacklist=blacklisted_rse_id)]
                                             except (InsufficientTargetRSEs, InsufficientAccountLimit, InvalidRuleWeight, RSEOverQuota) as error:
-                                                logging.error(prepend_str + 'Problem getting RSEs for subscription "%s" for account %s : %s. Skipping rule creation.' %
-                                                              (subscription['name'], account, str(error)))
-                                                monitor.record_counter(counters='transmogrifier.addnewrule.errortype.%s' % (str(error.__class__.__name__)), delta=1)
-                                                # The DID won't be reevaluated at the next cycle
-                                                did_success = did_success and True
-                                                continue
+                                                logging.warning(prepend_str + 'Problem getting RSEs for subscription "%s" for account %s : %s. Try including blacklisted sites' %
+                                                                (subscription['name'], account, str(error)))
+                                                # Now including the blacklisted sites
+                                                try:
+                                                    rseselector = RSESelector(account=account, rses=rses, weight=weight, copies=copies - len(preferred_rse_ids))
+                                                    selected_rses = [rse_id_dict[rse_id] for rse_id, _, _ in rseselector.select_rse(0, preferred_rse_ids=preferred_rse_ids, copies=copies, blacklist=[])]
+                                                    ignore_availability = True
+                                                except (InsufficientTargetRSEs, InsufficientAccountLimit, InvalidRuleWeight, RSEOverQuota) as error:
+                                                    logging.error(prepend_str + 'Problem getting RSEs for subscription "%s" for account %s : %s. Skipping rule creation.' %
+                                                                  (subscription['name'], account, str(error)))
+                                                    monitor.record_counter(counters='transmogrifier.addnewrule.errortype.%s' % (str(error.__class__.__name__)), delta=1)
+                                                    # The DID won't be reevaluated at the next cycle
+                                                    did_success = did_success and True
+                                                    continue
 
                                     for attempt in range(0, nattempt):
                                         attemptnr = attempt
@@ -310,21 +357,25 @@ def transmogrifier(bulk=5, once=False, sleep_time=60):
                                             if split_rule:
                                                 if not skip_rule_creation:
                                                     for rse in selected_rses:
+                                                        if isinstance(selected_rses, dict):
+                                                            source_replica_expression = selected_rses[rse].get('source_replica_expression', None)
+                                                            weight = selected_rses[rse].get('weight', None)
                                                         logging.info(prepend_str + 'Will insert one rule for %s:%s on %s' % (did['scope'], did['name'], rse))
-                                                        add_rule(dids=[{'scope': did['scope'], 'name': did['name']}], account=account, copies=1,
-                                                                 rse_expression=rse, grouping=grouping, weight=weight, lifetime=lifetime, locked=locked,
-                                                                 subscription_id=subscription_id, source_replica_expression=source_replica_expression, activity=activity,
-                                                                 purge_replicas=purge_replicas, ignore_availability=ignore_availability, comment=comment)
-
+                                                        rule_ids = add_rule(dids=[{'scope': did['scope'], 'name': did['name']}], account=account, copies=1,
+                                                                            rse_expression=rse, grouping=grouping, weight=weight, lifetime=lifetime, locked=locked,
+                                                                            subscription_id=subscription_id, source_replica_expression=source_replica_expression, activity=activity,
+                                                                            purge_replicas=purge_replicas, ignore_availability=ignore_availability, comment=comment)
+                                                        created_rules[cnt].append(rule_ids[0])
                                                         nb_rule += 1
                                                         if nb_rule == copies:
                                                             success = True
                                                             break
                                             else:
-                                                add_rule(dids=[{'scope': did['scope'], 'name': did['name']}], account=account, copies=copies,
-                                                         rse_expression=rse_expression, grouping=grouping, weight=weight, lifetime=lifetime, locked=locked,
-                                                         subscription_id=subscription['id'], source_replica_expression=source_replica_expression, activity=activity,
-                                                         purge_replicas=purge_replicas, ignore_availability=ignore_availability, comment=comment)
+                                                rule_ids = add_rule(dids=[{'scope': did['scope'], 'name': did['name']}], account=account, copies=copies,
+                                                                    rse_expression=rse_expression, grouping=grouping, weight=weight, lifetime=lifetime, locked=locked,
+                                                                    subscription_id=subscription['id'], source_replica_expression=source_replica_expression, activity=activity,
+                                                                    purge_replicas=purge_replicas, ignore_availability=ignore_availability, comment=comment)
+                                                created_rules[cnt].append(rule_ids[0])
                                                 nb_rule += 1
                                             monitor.record_counter(counters='transmogrifier.addnewrule.done', delta=nb_rule)
                                             monitor.record_counter(counters='transmogrifier.addnewrule.activity.%s' % str_activity, delta=nb_rule)


Test patch: