Repository: rucio/rucio

Issue:
Test new rule mode and switch it to default
Remove old rule mode from codebase


Discussion:

Resolving patch:
diff --git a/lib/rucio/core/rule.py b/lib/rucio/core/rule.py
--- a/lib/rucio/core/rule.py
+++ b/lib/rucio/core/rule.py
@@ -35,6 +35,7 @@
 # - Radu Carpa <radu.carpa@cern.ch>, 2021
 # - Rakshita Varadarajan <rakshitajps@gmail.com>, 2021
 # - Rahul Chauhan <omrahulchauhan@gmail.com>, 2021
+# - David Población Criado <david.poblacion.criado@cern.ch>, 2021
 # - Joel Dierkes <joel.dierkes@cern.ch>, 2021
 
 from __future__ import division
@@ -94,9 +95,6 @@
 from rucio.extensions.forecast import T3CModel
 
 
-# module variable to activate use of new rule algorithm (rucio.core.rule_grouping.apply_rule)
-USE_NEW_RULE_ALGORITHM = False
-
 REGION = make_region().configure('dogpile.cache.memcached',
                                  expiration_time=3600,
                                  arguments={'url': config_get('cache', 'url', False, '127.0.0.1:11211'),
@@ -145,11 +143,6 @@ def add_rule(dids, account, copies, rse_expression, grouping, weight, lifetime,
 
     grouping = {'ALL': RuleGrouping.ALL, 'NONE': RuleGrouping.NONE}.get(grouping, RuleGrouping.DATASET)
 
-    if USE_NEW_RULE_ALGORITHM:
-        use_new_rule_algorithm = True
-    else:
-        use_new_rule_algorithm = config_get('rules', 'use_new_rule_algorithm', default=False, session=session)
-
     with record_timer_block('rule.add_rule'):
         # 1. Resolve the rse_expression into a list of RSE-ids
         with record_timer_block('rule.add_rule.parse_rse_expression'):
@@ -323,41 +316,12 @@ def add_rule(dids, account, copies, rse_expression, grouping, weight, lifetime,
                 logger(logging.DEBUG, "Created rule %s for injection due to Split Container mode", str(new_rule.id))
                 continue
 
-            if use_new_rule_algorithm:
-                # 5. Apply the rule
-                with record_timer_block('rule.add_rule.apply_rule'):
-                    try:
-                        apply_rule(did, new_rule, [x['id'] for x in rses], [x['id'] for x in source_rses], rseselector, session=session)
-                    except IntegrityError as error:
-                        raise ReplicationRuleCreationTemporaryFailed(error.args[0])
-            else:
-                # 5. Resolve the did to its contents
-                with record_timer_block('rule.add_rule.resolve_dids_to_locks_replicas'):
-                    # Get all Replicas, not only the ones interesting for the rse_expression
-                    datasetfiles, locks, replicas, source_replicas = __resolve_did_to_locks_and_replicas(did=did,
-                                                                                                         nowait=False,
-                                                                                                         restrict_rses=[rse['id'] for rse in rses],
-                                                                                                         source_rses=[rse['id'] for rse in source_rses],
-                                                                                                         session=session)
-
-                sumfiles = sum([len(x['files']) for x in datasetfiles])
-                if sumfiles > 30000:
-                    logger(logging.WARNING, 'Rule %s for %s:%s involves %d files', str(new_rule.id), new_rule.scope, new_rule.name, sumfiles)
-
-                # 6. Apply the replication rule to create locks, replicas and transfers
-                with record_timer_block('rule.add_rule.create_locks_replicas_transfers'):
-                    try:
-                        __create_locks_replicas_transfers(datasetfiles=datasetfiles,
-                                                          locks=locks,
-                                                          replicas=replicas,
-                                                          source_replicas=source_replicas,
-                                                          rseselector=rseselector,
-                                                          rule=new_rule,
-                                                          preferred_rse_ids=[],
-                                                          source_rses=[rse['id'] for rse in source_rses],
-                                                          session=session)
-                    except IntegrityError as error:
-                        raise ReplicationRuleCreationTemporaryFailed(error.args[0])
+            # 5. Apply the rule
+            with record_timer_block('rule.add_rule.apply_rule'):
+                try:
+                    apply_rule(did, new_rule, [x['id'] for x in rses], [x['id'] for x in source_rses], rseselector, session=session)
+                except IntegrityError as error:
+                    raise ReplicationRuleCreationTemporaryFailed(error.args[0])
 
             if new_rule.locks_stuck_cnt > 0:
                 new_rule.state = RuleState.STUCK
@@ -380,11 +344,8 @@ def add_rule(dids, account, copies, rse_expression, grouping, weight, lifetime,
             # Add rule to History
             insert_rule_history(rule=new_rule, recent=True, longterm=True, session=session)
 
-            if use_new_rule_algorithm:
-                logger(logging.INFO, "Created rule %s [%d/%d/%d] with new algorithm for did %s:%s in state %s", str(new_rule.id), new_rule.locks_ok_cnt,
-                       new_rule.locks_replicating_cnt, new_rule.locks_stuck_cnt, new_rule.scope, new_rule.name, str(new_rule.state))
-            else:
-                logger(logging.INFO, "Created rule %s [%d/%d/%d] for did %s:%s in state %s", str(new_rule.id), new_rule.locks_ok_cnt, new_rule.locks_replicating_cnt, new_rule.locks_stuck_cnt, new_rule.scope, new_rule.name, str(new_rule.state))
+            logger(logging.INFO, "Created rule %s [%d/%d/%d] with new algorithm for did %s:%s in state %s", str(new_rule.id), new_rule.locks_ok_cnt,
+                   new_rule.locks_replicating_cnt, new_rule.locks_stuck_cnt, new_rule.scope, new_rule.name, str(new_rule.state))
 
     return rule_ids
 
@@ -658,11 +619,6 @@ def inject_rule(rule_id, session=None, logger=logging.log):
     :raises:           InvalidReplicationRule, InsufficientAccountLimit, InvalidRSEExpression, DataId, RSEOverQuota
     """
 
-    if USE_NEW_RULE_ALGORITHM:
-        use_new_rule_algorithm = True
-    else:
-        use_new_rule_algorithm = config_get('rules', 'use_new_rule_algorithm', default=False, session=session)
-
     try:
         rule = session.query(models.ReplicationRule).filter(models.ReplicationRule.id == rule_id).with_for_update(nowait=True).one()
     except NoResultFound:
@@ -737,37 +693,12 @@ def inject_rule(rule_id, session=None, logger=logging.log):
         except TypeError as error:
             raise InvalidObject(error.args)
 
-    if use_new_rule_algorithm:
-        # 4. Apply the rule
-        with record_timer_block('rule.add_rule.apply_rule'):
-            try:
-                apply_rule(did, rule, [x['id'] for x in rses], [x['id'] for x in source_rses], rseselector, session=session)
-            except IntegrityError as error:
-                raise ReplicationRuleCreationTemporaryFailed(error.args[0])
-    else:
-        # 5. Resolve the did to its contents
-        with record_timer_block('rule.add_rule.resolve_dids_to_locks_replicas'):
-            # Get all Replicas, not only the ones interesting for the rse_expression
-            datasetfiles, locks, replicas, source_replicas = __resolve_did_to_locks_and_replicas(did=did,
-                                                                                                 nowait=True,
-                                                                                                 restrict_rses=[rse['id'] for rse in rses],
-                                                                                                 source_rses=[rse['id'] for rse in source_rses],
-                                                                                                 session=session)
-
-        # 6. Apply the replication rule to create locks, replicas and transfers
-        with record_timer_block('rule.add_rule.create_locks_replicas_transfers'):
-            try:
-                __create_locks_replicas_transfers(datasetfiles=datasetfiles,
-                                                  locks=locks,
-                                                  replicas=replicas,
-                                                  source_replicas=source_replicas,
-                                                  rseselector=rseselector,
-                                                  rule=rule,
-                                                  preferred_rse_ids=[],
-                                                  source_rses=[rse['id'] for rse in source_rses],
-                                                  session=session)
-            except IntegrityError as error:
-                raise ReplicationRuleCreationTemporaryFailed(error.args[0])
+    # 4. Apply the rule
+    with record_timer_block('rule.add_rule.apply_rule'):
+        try:
+            apply_rule(did, rule, [x['id'] for x in rses], [x['id'] for x in source_rses], rseselector, session=session)
+        except IntegrityError as error:
+            raise ReplicationRuleCreationTemporaryFailed(error.args[0])
 
     if rule.locks_stuck_cnt > 0:
         rule.state = RuleState.STUCK
@@ -792,10 +723,7 @@ def inject_rule(rule_id, session=None, logger=logging.log):
     # Add rule to History
     insert_rule_history(rule=rule, recent=True, longterm=True, session=session)
 
-    if use_new_rule_algorithm:
-        logger(logging.INFO, "Created rule %s [%d/%d/%d] with new algorithm for did %s:%s in state %s", str(rule.id), rule.locks_ok_cnt, rule.locks_replicating_cnt, rule.locks_stuck_cnt, rule.scope, rule.name, str(rule.state))
-    else:
-        logger(logging.INFO, "Created rule %s [%d/%d/%d] for did %s:%s in state %s", str(rule.id), rule.locks_ok_cnt, rule.locks_replicating_cnt, rule.locks_stuck_cnt, rule.scope, rule.name, str(rule.state))
+    logger(logging.INFO, "Created rule %s [%d/%d/%d] with new algorithm for did %s:%s in state %s", str(rule.id), rule.locks_ok_cnt, rule.locks_replicating_cnt, rule.locks_stuck_cnt, rule.scope, rule.name, str(rule.state))
 
 
 @stream_session


Test patch:
diff --git a/lib/rucio/tests/test_rule_new_algorithm.py b/lib/rucio/tests/test_rule_new_algorithm.py
deleted file mode 100644
--- a/lib/rucio/tests/test_rule_new_algorithm.py
+++ /dev/null
@@ -1,38 +0,0 @@
-# -*- coding: utf-8 -*-
-# Copyright 2020-2021 CERN
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#    http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-# Authors:
-# - Luc Goossens <luc.goossens@cern.ch>, 2020
-# - Benedikt Ziemons <benedikt.ziemons@cern.ch>, 2021
-
-import pytest
-
-import rucio.core.rule
-from rucio.tests.test_rule import TestReplicationRuleCore
-
-
-@pytest.mark.noparallel(reason='sets a global variable')
-class TestReplicationRuleCoreNewAlgorithm(TestReplicationRuleCore):
-
-    @classmethod
-    def setUpClass(cls):
-        TestReplicationRuleCore.setUpClass()
-        rucio.core.rule.USE_NEW_RULE_ALGORITHM = True
-        print('set rucio.core.rule.USE_NEW_RULE_ALGORITHM to True')
-
-    @classmethod
-    def tearDownClass(cls):
-        rucio.core.rule.USE_NEW_RULE_ALGORITHM = False
-        print('set rucio.core.rule.USE_NEW_RULE_ALGORITHM back to False')