Repository: rucio/rucio

Issue:
add_rule on a constituent should make rule on archive
Motivation
----------
Internal forward of the rule to the archive.
How the callbacks are handled needs to be checked.


Discussion:

Resolving patch:
diff --git a/lib/rucio/core/rule.py b/lib/rucio/core/rule.py
--- a/lib/rucio/core/rule.py
+++ b/lib/rucio/core/rule.py
@@ -166,7 +166,29 @@ def add_rule(dids, account, copies, rse_expression, grouping, weight, lifetime,
                 except TypeError as error:
                     raise InvalidObject(error.args)
 
-            # 3.5 Get the lifetime
+            # 3.1 If the did is a constituent, relay the rule to the archive
+            if did.did_type == DIDType.FILE and did.constituent:
+                # Check if a single replica of this DID exists
+                replica_cnt = session.query(models.RSEFileAssociation).filter(models.RSEFileAssociation.scope == did.scope,
+                                                                              models.RSEFileAssociation.name == did.name,
+                                                                              models.RSEFileAssociation.state == ReplicaState.AVAILABLE).count()
+                if replica_cnt == 0:  # Put the rule on the archive
+                    archive = session.query(models.ConstituentAssociation).filter(models.ConstituentAssociation.child_scope == did.scope,
+                                                                                  models.ConstituentAssociation.child_name == did.name).first()
+                    if archive is not None:
+                        elem['name'] = archive.name
+                        elem['scope'] = archive.scope
+                        try:
+                            did = session.query(models.DataIdentifier).filter(models.DataIdentifier.scope == elem['scope'],
+                                                                              models.DataIdentifier.name == elem['name']).one()
+                        except NoResultFound:
+                            raise DataIdentifierNotFound('Data identifier %s:%s is not valid.' % (elem['scope'], elem['name']))
+                        except TypeError as error:
+                            raise InvalidObject(error.args)
+                else:  # Put the rule on the constituent directly
+                    pass
+
+            # 3.2 Get the lifetime
             eol_at = define_eol(elem['scope'], elem['name'], rses, session=session)
 
             # 4. Create the replication rule
@@ -343,7 +365,6 @@ def add_rules(dids, rules, session=None):
             all_source_rses = list(set([rse['id'] for rse in all_source_rses]))
 
         for elem in dids:
-            rule_ids[(elem['scope'], elem['name'])] = []
             # 2. Get the did
             with record_timer_block('rule.add_rules.get_did'):
                 try:
@@ -355,6 +376,29 @@ def add_rules(dids, rules, session=None):
                 except TypeError as error:
                     raise InvalidObject(error.args)
 
+            # 2.1 If the did is a constituent, relay the rule to the archive
+            if did.did_type == DIDType.FILE and did.constituent:  # Check if a single replica of this DID exists
+                replica_cnt = session.query(models.RSEFileAssociation).filter(models.RSEFileAssociation.scope == did.scope,
+                                                                              models.RSEFileAssociation.name == did.name,
+                                                                              models.RSEFileAssociation.state == ReplicaState.AVAILABLE).count()
+                if replica_cnt == 0:  # Put the rule on the archive
+                    archive = session.query(models.ConstituentAssociation).filter(models.ConstituentAssociation.child_scope == did.scope,
+                                                                                  models.ConstituentAssociation.child_name == did.name).first()
+                    if archive is not None:
+                        elem['name'] = archive.name
+                        elem['scope'] = archive.scope
+                        try:
+                            did = session.query(models.DataIdentifier).filter(models.DataIdentifier.scope == elem['scope'],
+                                                                              models.DataIdentifier.name == elem['name']).one()
+                        except NoResultFound:
+                            raise DataIdentifierNotFound('Data identifier %s:%s is not valid.' % (elem['scope'], elem['name']))
+                        except TypeError as error:
+                            raise InvalidObject(error.args)
+                else:  # Put the rule on the constituent directly
+                    pass
+
+            rule_ids[(elem['scope'], elem['name'])] = []
+
             # 3. Resolve the did into its contents
             with record_timer_block('rule.add_rules.resolve_dids_to_locks_replicas'):
                 # Get all Replicas, not only the ones interesting for the rse_expression


Test patch:
diff --git a/lib/rucio/tests/test_rule.py b/lib/rucio/tests/test_rule.py
--- a/lib/rucio/tests/test_rule.py
+++ b/lib/rucio/tests/test_rule.py
@@ -38,7 +38,7 @@
 from rucio.core.replica import add_replica, get_replica
 from rucio.core.rse import add_rse_attribute, get_rse, add_rse, update_rse, get_rse_id, del_rse_attribute
 from rucio.core.rse_counter import get_counter as get_rse_counter
-from rucio.core.rule import add_rule, get_rule, delete_rule, add_rules, update_rule, reduce_rule, move_rule
+from rucio.core.rule import add_rule, get_rule, delete_rule, add_rules, update_rule, reduce_rule, move_rule, list_rules
 from rucio.daemons.abacus.account import account_update
 from rucio.daemons.abacus.rse import rse_update
 from rucio.db.sqla import models
@@ -869,6 +869,35 @@ def test_metadata__rule(self):
                            weight='fakeweight', lifetime=None, locked=False, meta={'task_id': 55, 'job_ids': [1, 2, 3, 4]}, subscription_id=None)[0]
         assert(get_rule(rule_id)['meta'] == json.dumps({'task_id': 55, 'job_ids': [1, 2, 3, 4]}))
 
+    def test_rule_on_archive(self):
+        """ REPLICATION RULE (CORE): Test to add a rule on a constituent should add rule on archive"""
+        scope = 'mock'
+        archive = {'scope': scope, 'name': '%s.zip' % str(uuid()), 'type': 'FILE',
+                   'bytes': 2596, 'adler32': 'beefdead'}
+        add_replica(rse=self.rse1, scope=scope, name=archive['name'], bytes=2596, account='jdoe')
+        files_in_archive = [{'scope': scope, 'name': 'witrep-%i-%s' % (i, str(uuid())), 'type': 'FILE',
+                             'bytes': 1234, 'adler32': 'deadbeef'} for i in xrange(2)]
+        attach_dids(scope, archive['name'], files_in_archive, 'jdoe')
+
+        add_rule(dids=[{'scope': scope, 'name': files_in_archive[1]['name']}], account='jdoe', copies=1, rse_expression=self.rse3, grouping='NONE',
+                 weight=None, lifetime=None, locked=False, subscription_id=None)
+        assert(len(list(list_rules(filters={'scope': scope, 'name': archive['name']}))) == 1)
+
+        # Check the same but now a replica of the constituent exists as well
+        scope = 'mock'
+        archive = {'scope': scope, 'name': '%s.zip' % str(uuid()), 'type': 'FILE',
+                   'bytes': 2596, 'adler32': 'beefdead'}
+        add_replica(rse=self.rse1, scope=scope, name=archive['name'], bytes=2596, account='jdoe')
+        files_in_archive = [{'scope': scope, 'name': 'witrep-%i-%s' % (i, str(uuid())), 'type': 'FILE',
+                             'bytes': 1234, 'adler32': 'deadbeef'} for i in xrange(2)]
+        attach_dids(scope, archive['name'], files_in_archive, 'jdoe')
+        add_replica(rse=self.rse1, scope=scope, name=files_in_archive[1]['name'], bytes=2596, account='jdoe')
+
+        add_rule(dids=[{'scope': scope, 'name': files_in_archive[1]['name']}], account='jdoe', copies=1, rse_expression=self.rse3, grouping='NONE',
+                 weight=None, lifetime=None, locked=False, subscription_id=None)
+        assert(len(list(list_rules(filters={'scope': scope, 'name': archive['name']}))) == 0)
+        assert(len(list(list_rules(filters={'scope': scope, 'name': files_in_archive[1]['name']}))) == 1)
+
 
 class TestReplicationRuleClient():