Repository: rucio/rucio

Issue:
BB8 rewrite
Motivation
----------
BB8 has a lot of hardcoded values and is very ATLAS biased. Rewrite of some part of the code is needed




Discussion:

Resolving patch:
diff --git a/lib/rucio/core/rse.py b/lib/rucio/core/rse.py
--- a/lib/rucio/core/rse.py
+++ b/lib/rucio/core/rse.py
@@ -18,7 +18,7 @@
 # - Ralph Vigne <ralph.vigne@cern.ch>, 2012-2015
 # - Mario Lassnig <mario.lassnig@cern.ch>, 2012-2021
 # - Martin Barisits <martin.barisits@cern.ch>, 2013-2020
-# - Cedric Serfon <cedric.serfon@cern.ch>, 2013-2021
+# - Cedric Serfon <cedric.serfon@cern.ch>, 2013-2022
 # - Thomas Beermann <thomas.beermann@cern.ch>, 2014-2017
 # - Wen Guan <wen.guan@cern.ch>, 2015-2016
 # - Brian Bockelman <bbockelm@cse.unl.edu>, 2018
@@ -39,6 +39,7 @@
 # - Joel Dierkes <joel.dierkes@cern.ch>, 2021
 # - David Población Criado <david.poblacion.criado@cern.ch>, 2021
 
+from datetime import datetime
 import json
 from io import StringIO
 from re import match
@@ -51,7 +52,7 @@
 from sqlalchemy.exc import DatabaseError, IntegrityError, OperationalError
 from sqlalchemy.orm import aliased
 from sqlalchemy.orm.exc import FlushError
-from sqlalchemy.sql.expression import or_, false
+from sqlalchemy.sql.expression import or_, false, func, case
 
 import rucio.core.account_counter
 from rucio.common import exception, utils
@@ -60,7 +61,7 @@
 from rucio.common.utils import CHECKSUM_KEY, is_checksum_valid, GLOBALLY_SUPPORTED_CHECKSUMS
 from rucio.core.rse_counter import add_counter, get_counter
 from rucio.db.sqla import models
-from rucio.db.sqla.constants import RSEType
+from rucio.db.sqla.constants import (RSEType, ReplicaState)
 from rucio.db.sqla.session import read_session, transactional_session, stream_session
 
 if TYPE_CHECKING:
@@ -1398,3 +1399,27 @@ def list_qos_policies(rse_id, session=None):
         raise exception.RucioException(error.args)
 
     return qos_policies
+
+
+@transactional_session
+def fill_rse_expired(rse_id, session=None):
+    """
+    Fill the rse_usage for source expired
+
+    :param rse_id: The RSE id.
+
+    :returns: True if successful, except otherwise.
+    """
+    none_value = None  # Hack to get pep8 happy...
+    query = session.query(func.sum(models.RSEFileAssociation.bytes).label("bytes"), func.count().label("length")).\
+        with_hint(models.RSEFileAssociation, "INDEX_RS_ASC(replicas REPLICAS_TOMBSTONE_IDX)  NO_INDEX_FFS(replicas REPLICAS_TOMBSTONE_IDX)", 'oracle').\
+        filter(models.RSEFileAssociation.tombstone < datetime.utcnow()).\
+        filter(models.RSEFileAssociation.lock_cnt == 0).\
+        filter(case([(models.RSEFileAssociation.tombstone != none_value, models.RSEFileAssociation.rse_id), ]) == rse_id).\
+        filter(or_(models.RSEFileAssociation.state.in_((ReplicaState.AVAILABLE, ReplicaState.UNAVAILABLE, ReplicaState.BAD))))
+    result = query.all()
+    sum_bytes, sum_files = result[0]
+    models.RSEUsage(rse_id=rse_id,
+                    used=sum_bytes,
+                    files=sum_files,
+                    source='expired').save(session=session)
diff --git a/lib/rucio/daemons/bb8/bb8.py b/lib/rucio/daemons/bb8/bb8.py
--- a/lib/rucio/daemons/bb8/bb8.py
+++ b/lib/rucio/daemons/bb8/bb8.py
@@ -1,4 +1,5 @@
-# Copyright 2016-2018 CERN for the benefit of the ATLAS collaboration.
+# -*- coding: utf-8 -*-
+# Copyright 2016-2022 CERN
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -14,8 +15,9 @@
 #
 # Authors:
 # - Martin Barisits <martin.barisits@cern.ch>, 2016
-# - Cedric Serfon <cedric.serfon@cern.ch>, 2017
-# - Vincent Garonne <vgaronne@gmail.com>, 2018
+# - Cedric Serfon <cedric.serfon@cern.ch>, 2017-2022
+# - Vincent Garonne <vincent.garonne@cern.ch>, 2018
+# - Hannes Hansen <hannes.jakob.hansen@cern.ch>, 2018
 # - Thomas Beermann <thomas.beermann@cern.ch>, 2021
 
 """
@@ -25,33 +27,174 @@
 import logging
 import socket
 import threading
+import time
 import os
 
-from rucio.core.heartbeat import live, die, sanity_check
+from sqlalchemy import func, or_, and_
+from rucio.db.sqla.session import read_session
+
+from rucio.db.sqla import models
+from rucio.db.sqla.constants import RuleState, LockState
+from rucio.common.exception import InvalidRSEExpression
+from rucio.common.logging import formatted_logger, setup_logging
+from rucio.core import config as config_core
+from rucio.core.rse_expression_parser import parse_expression
+from rucio.core.heartbeat import live, die, sanity_check, list_payload_counts
+from rucio.core.rse import get_rse_usage
+from rucio.daemons.bb8.common import rebalance_rse
+
 
 GRACEFUL_STOP = threading.Event()
 
 
-def rule_rebalancer(once=False):
+def rule_rebalancer(rse_expression, move_subscriptions=False, use_dump=False, sleep_time=300, once=True, dry_run=False):
     """
     Main loop to rebalancer rules automatically
     """
 
-    raise NotImplementedError()
-
+    total_rebalance_volume = 0
+    executable = 'rucio-bb8'
     hostname = socket.gethostname()
     pid = os.getpid()
-    current_thread = threading.current_thread()
-
-    # Make an initial heartbeat so that all have the correct worker number on the next try
-    live(executable='rucio-bb8', hostname=hostname, pid=pid, thread=current_thread)
-    GRACEFUL_STOP.wait(1)
+    hb_thread = threading.current_thread()
+    heart_beat = live(executable, hostname, pid, hb_thread)
+    prepend_str = 'bb8[%i/%i] ' % (heart_beat['assign_thread'], heart_beat['nr_threads'])
+    logger = formatted_logger(logging.log, prepend_str + '%s')
+    logger(logging.DEBUG, 'rse_expression: %s', rse_expression)
+    logger(logging.INFO, 'BB8 started')
 
     while not GRACEFUL_STOP.is_set():
+        logger(logging.INFO, 'Starting new cycle')
+        heart_beat = live(executable, hostname, pid, hb_thread)
+        start_time = time.time()
+        total_rebalance_volume = 0
+        tolerance = config_core.get('bb8', 'tolerance', default=0.05)
+        max_total_rebalance_volume = config_core.get('bb8', 'max_total_rebalance_volume', default=10 * 1E12)
+        max_rse_rebalance_volume = config_core.get('bb8', 'max_rse_rebalance_volume', default=500 * 1E9)
+        min_total = config_core.get('bb8', 'min_total', default=20 * 1E9)
+        payload_cnt = list_payload_counts(executable, older_than=600, hash_executable=None, session=None)
+        if rse_expression in payload_cnt:
+            logger(logging.WARNING, 'One BB8 instance already running with the same RSE expression. Stopping')
+            break
+        else:
+            # List the RSEs represented by rse_expression
+            try:
+                rses = [rse for rse in parse_expression(rse_expression)]
+                list_rses2 = [rse['rse'] for rse in rses]
+            except InvalidRSEExpression as err:
+                logger(logging.ERROR, err)
+                break
+            # List the RSEs represented by all the RSE expressions stored in heartbeat payload
+            list_rses1 = []
+            for rse_exp in payload_cnt:
+                if rse_exp:
+                    list_rses1 = [rse['rse'] for rse in parse_expression(rse_exp)]
+            for rse in list_rses2:
+                if rse in list_rses1:
+                    logger(logging.WARNING, 'Overlapping RSE expressions %s vs %s. Stopping', rse_exp, rse_expression)
+                    break
+
+            logger(logging.INFO, 'Will process rebalancing on %s', rse_expression)
+            heart_beat = live(executable, hostname, pid, hb_thread, older_than=max(600, sleep_time), hash_executable=None, payload=rse_expression, session=None)
+            total_primary = 0
+            total_secondary = 0
+            total_total = 0
+            global_ratio = float(0)
+            for rse in rses:
+                logger(logging.DEBUG, 'Getting RSE usage on %s', rse['rse'])
+                rse_usage = get_rse_usage(rse_id=rse['id'])
+                usage_dict = {}
+                for item in rse_usage:
+                    # TODO Check last update
+                    usage_dict[item['source']] = {'used': item['used'], 'free': item['free'], 'total': item['total']}
+
+                try:
+                    rse['primary'] = usage_dict['rucio']['used'] - usage_dict['expired']['used']
+                    rse['secondary'] = usage_dict['expired']['used']
+                    rse['total'] = usage_dict['storage']['total'] - usage_dict['min_free_space']['used']
+                    rse['ratio'] = float(rse['primary']) / float(rse['total'])
+                except KeyError as err:
+                    logger(logging.ERROR, 'Missing source usage %s for RSE %s. Exiting', err, rse['rse'])
+                    break
+                total_primary += rse['primary']
+                total_secondary += rse['secondary']
+                total_total += float(rse['total'])
+                rse['receive_volume'] = 0  # Already rebalanced volume in this run
+                global_ratio = float(total_primary) / float(total_total)
+                logger(logging.INFO, 'Global ratio: %f' % (global_ratio))
+
+            for rse in sorted(rses, key=lambda k: k['ratio']):
+                logger(logging.INFO, '%s Sec/Prim local ratio (%f) vs global %s', rse['rse'], rse['ratio'], global_ratio)
+            rses_over_ratio = sorted([rse for rse in rses if rse['ratio'] > global_ratio + global_ratio * tolerance], key=lambda k: k['ratio'], reverse=True)
+            rses_under_ratio = sorted([rse for rse in rses if rse['ratio'] < global_ratio - global_ratio * tolerance], key=lambda k: k['ratio'], reverse=False)
+
+            # Excluding RSEs
+            logger(logging.DEBUG, 'Excluding RSEs as destination which are too small by size:')
+            for des in rses_under_ratio:
+                if des['total'] < min_total:
+                    logger(logging.DEBUG, 'Excluding %s', des['rse'])
+                    rses_under_ratio.remove(des)
+            logger(logging.DEBUG, 'Excluding RSEs as sources which are too small by size:')
+            for src in rses_over_ratio:
+                if src['total'] < min_total:
+                    logger(logging.DEBUG, 'Excluding %s', src['rse'])
+                    rses_over_ratio.remove(src)
+            logger(logging.DEBUG, 'Excluding RSEs as destinations which are not available for write:')
+            for des in rses_under_ratio:
+                if des['availability'] & 2 == 0:
+                    logger(logging.DEBUG, 'Excluding %s', des['rse'])
+                    rses_under_ratio.remove(des)
+            logger(logging.DEBUG, 'Excluding RSEs as sources which are not available for read:')
+            for src in rses_over_ratio:
+                if src['availability'] & 4 == 0:
+                    logger(logging.DEBUG, 'Excluding %s', src['rse'])
+                    rses_over_ratio.remove(src)
+
+            # Gets the number of active transfers per location
+            dict_locks = get_active_locks(session=None)
+
+            # Loop over RSEs over the ratio
+            for index, source_rse in enumerate(rses_over_ratio):
+
+                # The volume that would be rebalanced, not real availability of the data:
+                available_source_rebalance_volume = int((source_rse['primary'] - global_ratio * source_rse['secondary']) / (global_ratio + 1))
+                if available_source_rebalance_volume > max_rse_rebalance_volume:
+                    available_source_rebalance_volume = max_rse_rebalance_volume
+                if available_source_rebalance_volume > max_total_rebalance_volume - total_rebalance_volume:
+                    available_source_rebalance_volume = max_total_rebalance_volume - total_rebalance_volume
+
+                # Select a target:
+                for destination_rse in rses_under_ratio:
+                    if available_source_rebalance_volume > 0:
+                        vo_str = ' on VO {}'.format(destination_rse['vo']) if destination_rse['vo'] != 'def' else ''
+                        if index == 0 and destination_rse['id'] in dict_locks:
+                            replicating_volume = dict_locks[destination_rse['id']]['bytes']
+                            logger(logging.DEBUG, 'Already %f TB replicating to %s%s', replicating_volume / 1E12, destination_rse['rse'], vo_str)
+                            destination_rse['receive_volume'] += replicating_volume
+                        if destination_rse['receive_volume'] >= max_rse_rebalance_volume:
+                            continue
+                        available_target_rebalance_volume = max_rse_rebalance_volume - destination_rse['receive_volume']
+                        if available_target_rebalance_volume >= available_source_rebalance_volume:
+                            available_target_rebalance_volume = available_source_rebalance_volume
+
+                        logger(logging.INFO, 'Rebalance %d TB from %s(%f) to %s(%f)%s', available_target_rebalance_volume / 1E12, source_rse['rse'], source_rse['ratio'], destination_rse['rse'], destination_rse['ratio'], vo_str)
+                        expr = destination_rse['rse']
+                        rebalance_rse(rse_id=source_rse['id'], max_bytes=available_target_rebalance_volume, dry_run=dry_run, comment='Background rebalancing', force_expression=expr, logger=logger)
+
+                        destination_rse['receive_volume'] += available_target_rebalance_volume
+                        total_rebalance_volume += available_target_rebalance_volume
+                        available_source_rebalance_volume -= available_target_rebalance_volume
+
         if once:
             break
 
-    die(executable='rucio-bb8', hostname=hostname, pid=pid, thread=current_thread)
+        end_time = time.time()
+        time_diff = end_time - start_time
+        if time_diff < sleep_time:
+            logger(logging.INFO, 'Sleeping for a while : %f seconds', sleep_time - time_diff)
+            GRACEFUL_STOP.wait(sleep_time - time_diff)
+
+    die(executable='rucio-bb8', hostname=hostname, pid=pid, thread=hb_thread)
 
 
 def stop(signum=None, frame=None):
@@ -59,27 +202,42 @@ def stop(signum=None, frame=None):
     Graceful exit.
     """
 
-    raise NotImplementedError()
-
     GRACEFUL_STOP.set()
 
 
-def run(once=False, threads=1):
+def run(once, rse_expression, move_subscriptions=False, use_dump=False, sleep_time=300, threads=1, dry_run=False):
     """
-    Starts up the Judge-Clean threads.
+    Starts up the BB8 rebalancing threads.
     """
 
-    raise NotImplementedError()
-
+    setup_logging()
     hostname = socket.gethostname()
     sanity_check(executable='rucio-bb8', hostname=hostname)
 
     if once:
-        rule_rebalancer(once)
+        rule_rebalancer(rse_expression=rse_expression, move_subscriptions=move_subscriptions, use_dump=use_dump, once=once)
     else:
-        logging.info('BB8 starting %s threads' % str(threads))
-        threads = [threading.Thread(target=rule_rebalancer, kwargs={'once': once}) for _ in range(0, threads)]
-        [t.start() for t in threads]
+        logging.info('BB8 starting %s threads', str(threads))
+        threads = [threading.Thread(target=rule_rebalancer, kwargs={'once': once, 'rse_expression': rse_expression, 'sleep_time': sleep_time, 'dry_run': dry_run}) for _ in range(0, threads)]
+        [thread.start() for thread in threads]
         # Interruptible joins require a timeout.
         while threads[0].is_alive():
-            [t.join(timeout=3.14) for t in threads]
+            [thread.join(timeout=3.14) for thread in threads]
+
+
+@read_session
+def get_active_locks(session=None):
+    locks_dict = {}
+    rule_ids = session.query(models.ReplicationRule.id).filter(or_(models.ReplicationRule.state == RuleState.REPLICATING, models.ReplicationRule.state == RuleState.STUCK),
+                                                               models.ReplicationRule.comments == 'Background rebalancing').all()
+    for row in rule_ids:
+        rule_id = row[0]
+        query = session.query(func.count(), func.sum(models.ReplicaLock.bytes), models.ReplicaLock.state, models.ReplicaLock.rse_id).\
+            filter(and_(models.ReplicaLock.rule_id == rule_id, models.ReplicaLock.state != LockState.OK)).group_by(models.ReplicaLock.state, models.ReplicaLock.rse_id)
+        for lock in query.all():
+            cnt, size, _, rse_id = lock
+            if rse_id not in locks_dict:
+                locks_dict[rse_id] = {'bytes': 0, 'locks': 0}
+            locks_dict[rse_id]['locks'] += cnt
+            locks_dict[rse_id]['bytes'] += size
+    return locks_dict
diff --git a/lib/rucio/daemons/bb8/common.py b/lib/rucio/daemons/bb8/common.py
--- a/lib/rucio/daemons/bb8/common.py
+++ b/lib/rucio/daemons/bb8/common.py
@@ -1,5 +1,5 @@
 # -*- coding: utf-8 -*-
-# Copyright 2016-2021 CERN
+# Copyright 2016-2022 CERN
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -17,20 +17,23 @@
 # - Martin Barisits <martin.barisits@cern.ch>, 2016-2021
 # - Vincent Garonne <vincent.garonne@cern.ch>, 2016-2018
 # - Tomas Javurek <tomas.javurek@cern.ch>, 2017-2019
-# - Cedric Serfon <cedric.serfon@cern.ch>, 2017
+# - Cedric Serfon <cedric.serfon@cern.ch>, 2017-2022
 # - Dimitrios Christidis <dimitrios.christidis@cern.ch>, 2018-2020
 # - Hannes Hansen <hannes.jakob.hansen@cern.ch>, 2018
 # - Andrew Lister <andrew.lister@stfc.ac.uk>, 2019
 # - Eli Chadwick <eli.chadwick@stfc.ac.uk>, 2020
 # - Patrick Austin <patrick.austin@stfc.ac.uk>, 2020
 # - Thomas Beermann <thomas.beermann@cern.ch>, 2021
+# - David Población Criado <david.poblacion.criado@cern.ch>, 2021
 
 from __future__ import print_function, division
 
+import logging
+
 from datetime import datetime, date, timedelta
 from string import Template
 from sqlalchemy.orm import aliased
-from sqlalchemy import func, and_, or_, cast, Integer
+from sqlalchemy import func, and_, or_, cast, BigInteger
 from sqlalchemy.sql.expression import case, select
 
 from rucio.core.lock import get_dataset_locks
@@ -38,21 +41,21 @@
 from rucio.core.rse_expression_parser import parse_expression
 from rucio.core.rse import list_rse_attributes, get_rse_name, get_rse_vo
 from rucio.core.rse_selector import RSESelector
-from rucio.common.config import config_get
+from rucio.common.config import config_get, config_get_int, config_get_bool
 from rucio.common.exception import (InsufficientTargetRSEs, RuleNotFound, DuplicateRule,
                                     InsufficientAccountLimit)
 from rucio.common.types import InternalAccount
 
-from rucio.db.sqla.session import transactional_session
+from rucio.db.sqla.session import transactional_session, read_session
 from rucio.db.sqla import models
 from rucio.db.sqla.constants import (DIDType, RuleState, RuleGrouping)
 from requests import get
 
 
-def rebalance_rule(parent_rule, activity, rse_expression, priority, source_replica_expression='*\\bb8-enabled=false', comment=None):
+@transactional_session
+def rebalance_rule(parent_rule, activity, rse_expression, priority, source_replica_expression='*\\bb8-enabled=false', comment=None, session=None):
     """
     Rebalance a replication rule to a new RSE
-
     :param parent_rule:                Replication rule to be rebalanced.
     :param activity:                   Activity to be used for the rebalancing.
     :param rse_expression:             RSE expression of the new rule.
@@ -76,18 +79,13 @@ def rebalance_rule(parent_rule, activity, rse_expression, priority, source_repli
 
     # check if concurrent replica at target rse does not exist
     concurrent_replica = False
-    try:
-        for lock in get_dataset_locks(parent_rule['scope'], parent_rule['name']):
-            lock_rse_expr = lock['rse']
-            if lock_rse_expr == rse_expression:
-                concurrent_replica = True
-    except Exception as error:
-        concurrent_replica = True
-        print('Exception: get_dataset_locks not feasible for %s %s:' % (parent_rule['scope'], parent_rule['name']))
-        raise error
+    for lock in get_dataset_locks(parent_rule['scope'], parent_rule['name']):
+        lock_rse_expr = lock['rse']
+        if lock_rse_expr == rse_expression:
+            concurrent_replica = True
+
     if concurrent_replica:
-        return 'Concurrent replica exists at target rse!'
-    print(concurrent_replica)
+        return None
 
     child_rule = add_rule(dids=[{'scope': parent_rule['scope'],
                                  'name': parent_rule['name']}],
@@ -108,17 +106,17 @@ def rebalance_rule(parent_rule, activity, rse_expression, priority, source_repli
                           ask_approval=False,
                           asynchronous=False,
                           ignore_account_limit=True,
-                          priority=priority)[0]
+                          priority=priority, session=session)[0]
 
-    update_rule(rule_id=parent_rule['id'], options={'child_rule_id': child_rule, 'lifetime': 0})
+    update_rule(rule_id=parent_rule['id'], options={'child_rule_id': child_rule, 'lifetime': 0}, session=session)
     return child_rule
 
 
-def __dump_url(rse_id):
+def __dump_url(rse_id, logger=logging.log):
     """
     getting potential urls of the dump over last week
-
-    :param rse_id: RSE where the dump is released.
+    :param rse_id:                     RSE where the dump is released.
+    :param logger:                     Logger.
     """
 
     rse = get_rse_name(rse_id=rse_id)
@@ -135,7 +133,7 @@ def __dump_url(rse_id):
     else:
         weekdays = {'Sunday': 6, 'Monday': 0, 'Tuesday': 1, 'Wednesday': 2, 'Thursday': 3, 'Friday': 4, 'Saturday': 5}
         if dump_production_day not in weekdays:
-            print('ERROR: please set the day of a dump creation in bb8 config correctly, e.g. Monday')
+            logger(logging.WARNING, 'ERROR: please set the day of a dump creation in bb8 config correctly, e.g. Monday')
             return False
         today_idx = (today.weekday() - weekdays[dump_production_day]) % 7
         dump_date = today - timedelta(today_idx)
@@ -153,12 +151,12 @@ def __dump_url(rse_id):
     return urls
 
 
-def _list_rebalance_rule_candidates_dump(rse_id, mode=None):
+def _list_rebalance_rule_candidates_dump(rse_id, mode=None, logger=logging.log):
     """
-    Download dump to tmporary directory
-
-    :param rse_id:       RSE of the source.
-    :param mode:         Rebalancing mode.
+    Download dump to temporary directory
+    :param rse_id:                     RSE of the source.
+    :param mode:                       Rebalancing mode.
+    :param logger:                     Logger.
     """
 
     # fetching the dump
@@ -166,32 +164,31 @@ def _list_rebalance_rule_candidates_dump(rse_id, mode=None):
     rules = {}
     rse_dump_urls = __dump_url(rse_id=rse_id)
     rse_dump_urls.reverse()
-    r = None
+    resp = None
     if not rse_dump_urls:
-        print('URL of the dump was not built from template.')
+        logger(logging.DEBUG, 'URL of the dump was not built from template.')
         return candidates
     success = False
     while not success and len(rse_dump_urls):
         url = rse_dump_urls.pop()
-        print(url)
-        r = get(url, stream=True)
-        if r:
+        resp = get(url, stream=True)
+        if resp:
             success = True
-    if not r or r is None:
-        print('RSE dump not available')
+    if not resp or resp is None:
+        logger(logging.WARNING, 'RSE dump not available')
         return candidates
 
     # looping over the dump and selecting the rules
-    for line in r.iter_lines():
+    for line in resp.iter_lines():
         if line:
-            file_scope, file_name, rule_id, rse_expression, account, file_size, state = line.split('\t')
+            _, _, rule_id, rse_expression, account, file_size, state = line.split('\t')
             if rule_id not in rules:
                 rule_info = {}
                 try:
                     rule_info = get_rule(rule_id=rule_id)
-                except Exception as e:
+                except Exception as err:
                     rules[rule_id] = {'state': 'DELETED'}
-                    print(e)
+                    logger(logging.ERROR, str(err))
                     continue
                 if rule_info['child_rule_id']:
                     rules[rule_id] = {'state': 'DELETED'}
@@ -229,7 +226,6 @@ def _list_rebalance_rule_candidates_dump(rse_id, mode=None):
 def list_rebalance_rule_candidates(rse_id, mode=None, session=None):
     """
     List the rebalance rule candidates based on the agreed on specification
-
     :param rse_id:       RSE of the source.
     :param mode:         Rebalancing mode.
     :param session:      DB Session.
@@ -241,11 +237,68 @@ def list_rebalance_rule_candidates(rse_id, mode=None, session=None):
     if mode == 'decommission':
         return _list_rebalance_rule_candidates_dump(rse_id, mode)
 
-    # the rest is done with sql query
-    from_date = datetime.utcnow() + timedelta(days=60)
-    to_date = datetime.now() - timedelta(days=60)
-    allowed_accounts = [InternalAccount(a, vo=vo) for a in ('panda', 'root', 'ddmadmin')]
-    allowed_grouping = [RuleGrouping.DATASET, RuleGrouping.ALL]
+    # If no decommissioning use SQLAlchemy
+
+    # Rules constraints. By default only moves rules in state OK that have no children and have only one copy
+    # Additional constraints can be imposed by setting specific configuration
+    rule_clause = [models.ReplicationRule.state == RuleState.OK,
+                   models.ReplicationRule.child_rule_id.is_(None),
+                   models.ReplicationRule.copies == 1]
+
+    # Only move rules w/o expiration date, or rules with expiration_date > >min_expires_date_in_days> days
+    expiration_clause = models.ReplicationRule.expires_at.is_(None)
+    min_expires_date_in_days = config_get_int(section='bb8', option='min_expires_date_in_days', raise_exception=False, default=-1, expiration_time=3600)
+    if min_expires_date_in_days > 0:
+        min_expires_date_in_days = datetime.utcnow() + timedelta(days=min_expires_date_in_days)
+        expiration_clause = or_(models.ReplicationRule.expires_at > min_expires_date_in_days, models.ReplicationRule.expires_at.is_(None))
+    rule_clause.append(expiration_clause)
+
+    # Only move rules which were created more than <min_created_days> days ago
+    min_created_days = config_get_int(section='bb8', option='min_created_days', raise_exception=False, default=-1, expiration_time=3600)
+    if min_created_days > 0:
+        min_created_days = datetime.now() - timedelta(days=min_created_days)
+        rule_clause.append(models.ReplicationRule.created_at < min_created_days)
+
+    # Only move rules which are owned by <allowed_accounts> (coma separated accounts, e.g. panda,root,ddmadmin,jdoe)
+    allowed_accounts = config_get(section='bb8', option='allowed_accounts', raise_exception=False, default=None, expiration_time=3600)
+    if allowed_accounts:
+        allowed_accounts = [InternalAccount(acc.strip(' '), vo=vo) for acc in allowed_accounts.split(',')]
+        rule_clause.append(models.ReplicationRule.account.in_(allowed_accounts))
+
+    # Only move rules that have a certain grouping <allowed_grouping> (accepted values : all, dataset, none)
+    rule_grouping_mapping = {
+        'all': RuleGrouping.ALL,
+        'dataset': RuleGrouping.DATASET,
+        'none': RuleGrouping.NONE
+    }
+    allowed_grouping = config_get(section='bb8', option='allowed_grouping', raise_exception=False, default=None, expiration_time=3600)
+    if allowed_grouping:
+        rule_clause.append(models.ReplicationRule.grouping == rule_grouping_mapping.get(allowed_grouping))
+
+    # DIDs constraints. By default only moves rules of DID where we can compute the size
+    # Additional constraints can be imposed by setting specific configuration
+    did_clause = [models.DataIdentifier.bytes.isnot(None)]
+
+    type_to_did_type_mapping = {
+        'all': [DIDType.CONTAINER, DIDType.DATASET, DIDType.FILE],
+        'collection': [DIDType.CONTAINER, DIDType.DATASET],
+        'container': [DIDType.CONTAINER],
+        'dataset': [DIDType.DATASET],
+        'file': [DIDType.FILE]
+    }
+
+    # Only allows to migrate rules of a certain did_type <allowed_did_type> (accepted values : all, collection, container, dataset, file)
+    allowed_did_type = config_get(section='bb8', option='allowed_did_type', raise_exception=False, default=None, expiration_time=3600)
+    if allowed_did_type:
+        allowed_did_type = [models.DataIdentifier.did_type == did_type for did_type in type_to_did_type_mapping.get(allowed_did_type)]
+        did_clause.append(or_(allowed_did_type))
+
+    # Only allows to migrate rules of closed DID is <only_move_closed_did> is set
+    only_move_closed_did = config_get_bool(section='bb8', option='only_move_closed_did', raise_exception=False, default=None, expiration_time=3600)
+    if only_move_closed_did:
+        did_clause.append(models.DataIdentifier.is_open == False)  # NOQA
+
+    # Now build the query
     external_dsl = aliased(models.DatasetLock)
     count_locks = select([func.count()]).where(and_(external_dsl.scope == models.DatasetLock.scope,
                                                     external_dsl.name == models.DatasetLock.name,
@@ -258,35 +311,25 @@ def list_rebalance_rule_candidates(rse_id, mode=None, session=None):
                           models.DataIdentifier.bytes,
                           models.DataIdentifier.length,
                           case([(or_(models.DatasetLock.length < 1, models.DatasetLock.length.is_(None)), 0)],
-                               else_=cast(models.DatasetLock.bytes / models.DatasetLock.length, Integer))).\
+                               else_=cast(models.DatasetLock.bytes / models.DatasetLock.length, BigInteger))).\
         join(models.ReplicationRule, models.ReplicationRule.id == models.DatasetLock.rule_id).\
         join(models.DataIdentifier, and_(models.DatasetLock.scope == models.DataIdentifier.scope, models.DatasetLock.name == models.DataIdentifier.name)).\
         filter(models.DatasetLock.rse_id == rse_id).\
-        filter(or_(models.ReplicationRule.expires_at > from_date, models.ReplicationRule.expires_at.is_(None))).\
-        filter(models.ReplicationRule.created_at < to_date).\
-        filter(models.ReplicationRule.account.in_(allowed_accounts)).\
-        filter(models.ReplicationRule.state == RuleState.OK).\
-        filter(models.ReplicationRule.did_type == DIDType.DATASET).\
-        filter(models.ReplicationRule.copies == 1).\
-        filter(models.ReplicationRule.child_rule_id.is_(None)).\
-        filter(models.ReplicationRule.grouping.in_(allowed_grouping)).\
-        filter(models.DataIdentifier.bytes.isnot(None)).\
-        filter(models.DataIdentifier.is_open == 0).\
-        filter(models.DataIdentifier.did_type == DIDType.DATASET).\
+        filter(and_(*rule_clause)).\
+        filter(and_(*did_clause)).\
         filter(case([(or_(models.DatasetLock.length < 1, models.DatasetLock.length.is_(None)), 0)],
-                    else_=cast(models.DatasetLock.bytes / models.DatasetLock.length, Integer)) > 1000000000).\
+                    else_=cast(models.DatasetLock.bytes / models.DatasetLock.length, BigInteger)) > 1000000000).\
         filter(count_locks == 1)
     summary = query.order_by(case([(or_(models.DatasetLock.length < 1, models.DatasetLock.length.is_(None)), 0)],
-                                  else_=cast(models.DatasetLock.bytes / models.DatasetLock.length, Integer)),
+                                  else_=cast(models.DatasetLock.bytes / models.DatasetLock.length, BigInteger)),
                              models.DatasetLock.accessed_at).all()
     return summary
 
 
-@transactional_session
+@read_session
 def select_target_rse(parent_rule, current_rse_id, rse_expression, subscription_id, rse_attributes, other_rses=[], exclude_expression=None, force_expression=None, session=None):
     """
     Select a new target RSE for a rebalanced rule.
-
     :param parent_rule           rule that is rebalanced.
     :param current_rse_id:       RSE of the source.
     :param rse_expression:       RSE Expression of the source rule.
@@ -295,58 +338,48 @@ def select_target_rse(parent_rule, current_rse_id, rse_expression, subscription_
     :param other_rses:           Other RSEs with existing dataset replicas.
     :param exclude_expression:   Exclude this rse_expression from being target_rses.
     :param force_expression:     Force a specific rse_expression as target.
-    :param session:              The DB Session
-    :returns:                    New RSE expression
+    :param session:              The DB Session.
+    :returns:                    New RSE expression.
     """
 
-    if rse_attributes['type'] != 'DATADISK' and force_expression is None:
-        print('WARNING: dest RSE(s) has to be provided with --force-expression for rebalancing of non-datadisk RSES.')
-        raise InsufficientTargetRSEs
-
     current_rse = get_rse_name(rse_id=current_rse_id)
     current_rse_expr = current_rse
     # if parent rule has a vo, enforce it
     vo = parent_rule['scope'].vo
     if exclude_expression:
-        target_rse = '(%s)\\%s' % (exclude_expression, current_rse_expr)
+        target_rse = '((%s)|(%s))' % (exclude_expression, current_rse_expr)
     else:
         target_rse = current_rse_expr
+    list_target_rses = [rse['rse'] for rse in parse_expression(expression=target_rse, filter_={'vo': vo}, session=session)]
+    list_target_rses.sort()
 
     rses = parse_expression(expression=rse_expression, filter_={'vo': vo}, session=session)
-    # TODO: dest rse selection should be configurable, there might be cases when tier is not defined, or concept of DATADISKS is not present.
-    # if subscription_id:
-    #     pass
-    #     # get_subscription_by_id(subscription_id, session)
+
+    # TODO: Implement subscription rebalancing
+
     if force_expression is not None:
         if parent_rule['grouping'] != RuleGrouping.NONE:
             rses = parse_expression(expression='(%s)\\%s' % (force_expression, target_rse), filter_={'vo': vo, 'availability_write': True}, session=session)
         else:
-            # in order to avoid replication of the part of distributed dataset not present at rabalanced rse -> rses in force_expression
+            # in order to avoid replication of the part of distributed dataset not present at rebalanced rse -> rses in force_expression
             # this will be extended with development of delayed rule
             rses = parse_expression(expression='((%s)|(%s))\\%s' % (force_expression, rse_expression, target_rse), filter_={'vo': vo, 'availability_write': True}, session=session)
-    elif len(rses) > 1:
-        # Just define the RSE Expression without the current_rse
-        return '(%s)\\%s' % (rse_expression, target_rse)
     else:
-        if rse_attributes['tier'] is True or int(rse_attributes['tier']) == 1:
-            # Tier 1 should go to another Tier 1
-            expression = '(tier=1&type=DATADISK)\\{}'.format(target_rse)
-        elif int(rse_attributes['tier']) == 2:
-            # Tier 2 should go to another Tier 2
-            expression = '(tier=2&type=DATADISK)\\{}'.format(target_rse)
-        elif int(rse_attributes['tier']) == 3:
-            # Tier 3 will go to Tier 2, since we don't have enough t3s
-            expression = '((tier=2&type=DATADISK)\\datapolicynucleus=1)\\{}'.format(target_rse)
-        rses = parse_expression(expression=expression, filter_={'vo': vo, 'availability_write': True}, session=session)
-    rseselector = RSESelector(account=InternalAccount('ddmadmin', vo=vo), rses=rses, weight='freespace', copies=1, ignore_account_limit=True, session=session)
+        # force_expression is not set, RSEs will be selected as rse_expression\rse
+        list_rses = [rse['rse'] for rse in rses]
+        list_rses.sort()
+        if list_rses == list_target_rses:
+            raise InsufficientTargetRSEs('Not enough RSEs to rebalance rule %s' % parent_rule['id'])
+        else:
+            rses = parse_expression(expression='(%s)\\%s' % (rse_expression, target_rse), filter_={'vo': vo, 'availability_write': True}, session=session)
+    rseselector = RSESelector(account=InternalAccount('root', vo=vo), rses=rses, weight='freespace', copies=1, ignore_account_limit=True, session=session)
     return get_rse_name([rse_id for rse_id, _, _ in rseselector.select_rse(size=0, preferred_rse_ids=[], blocklist=other_rses)][0], session=session)
 
 
 @transactional_session
-def rebalance_rse(rse_id, max_bytes=1E9, max_files=None, dry_run=False, exclude_expression=None, comment=None, force_expression=None, mode=None, priority=3, source_replica_expression='*\\bb8-enabled=false', session=None):
+def rebalance_rse(rse_id, max_bytes=1E9, max_files=None, dry_run=False, exclude_expression=None, comment=None, force_expression=None, mode=None, priority=3, source_replica_expression='*\\bb8-enabled=false', session=None, logger=logging.log):
     """
     Rebalance data from an RSE
-
     :param rse_id:                     RSE to rebalance data from.
     :param max_bytes:                  Maximum amount of bytes to rebalance.
     :param max_files:                  Maximum amount of files to rebalance.
@@ -358,6 +391,7 @@ def rebalance_rse(rse_id, max_bytes=1E9, max_files=None, dry_run=False, exclude_
     :param priority:                   Priority of the new created rules.
     :param source_replica_expression:  Source replica expression of the new created rules.
     :param session:                    The database session.
+    :param logger:                     Logger.
     :returns:                          List of rebalanced datasets.
     """
     rebalanced_bytes = 0
@@ -365,14 +399,13 @@ def rebalance_rse(rse_id, max_bytes=1E9, max_files=None, dry_run=False, exclude_
     rebalanced_datasets = []
 
     rse_attributes = list_rse_attributes(rse_id=rse_id, session=session)
+    src_rse = get_rse_name(rse_id=rse_id)
 
-    print('***************************')
-    print('BB8 - Execution Summary')
-    print('Mode:    %s' % ('STANDARD' if mode is None else mode.upper()))
-    print('Dry Run: %s' % (dry_run))
-    print('***************************')
-
-    print('scope:name rule_id bytes(Gb) target_rse child_rule_id')
+    logger(logging.INFO, '***************************')
+    logger(logging.INFO, 'BB8 - Execution Summary')
+    logger(logging.INFO, 'Mode:    %s' % ('STANDARD' if mode is None else mode.upper()))
+    logger(logging.INFO, 'Dry Run: %s' % (dry_run))
+    logger(logging.INFO, '***************************')
 
     for scope, name, rule_id, rse_expression, subscription_id, bytes_, length, fsize in list_rebalance_rule_candidates(rse_id=rse_id, mode=mode):
         if force_expression is not None and subscription_id is not None:
@@ -408,18 +441,18 @@ def rebalance_rse(rse_id, max_bytes=1E9, max_files=None, dry_run=False, exclude_
                                                    comment=comment)
                 else:
                     child_rule_id = ''
-            except (InsufficientTargetRSEs, DuplicateRule, RuleNotFound, InsufficientAccountLimit):
+            except (InsufficientTargetRSEs, DuplicateRule, RuleNotFound, InsufficientAccountLimit) as err:
+                logger(logging.ERROR, str(err))
                 continue
-            print('%s:%s %s %d %s %s' % (scope, name, str(rule_id), int(bytes_ / 1E9), target_rse_exp, child_rule_id))
-            if 'Concurrent' in str(child_rule_id):
-                print(str(child_rule_id))
+            if child_rule_id is None:
+                logger(logging.WARNING, 'A rule for %s:%s already exists on %s. It cannot be rebalanced', scope, name, target_rse_exp)
                 continue
+            logger(logging.INFO, 'Rebalancing %s:%s rule %s (%f GB) from %s to %s. New rule %s', scope, name, str(rule_id), bytes_ / 1E9, rule['rse_expression'], target_rse_exp, child_rule_id)
             rebalanced_bytes += bytes_
             rebalanced_files += length
             rebalanced_datasets.append((scope, name, bytes_, length, target_rse_exp, rule_id, child_rule_id))
         except Exception as error:
-            print('Exception %s occured while rebalancing %s:%s, rule_id: %s!' % (str(error), scope, name, str(rule_id)))
-            raise error
+            logger(logging.ERROR, 'Exception %s occured while rebalancing %s:%s, rule_id: %s!', str(error), scope, name, str(rule_id))
 
-    print('BB8 is rebalancing %d Gb of data (%d rules)' % (int(rebalanced_bytes / 1E9), len(rebalanced_datasets)))
+    logger(logging.INFO, 'BB8 is rebalancing %d GB of data (%d rules) from %s', rebalanced_bytes / 1E9, len(rebalanced_datasets), src_rse)
     return rebalanced_datasets


Test patch:
diff --git a/lib/rucio/tests/conftest.py b/lib/rucio/tests/conftest.py
--- a/lib/rucio/tests/conftest.py
+++ b/lib/rucio/tests/conftest.py
@@ -1,5 +1,5 @@
 # -*- coding: utf-8 -*-
-# Copyright 2020-2021 CERN
+# Copyright 2020-2022 CERN
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -16,10 +16,11 @@
 # Authors:
 # - Benedikt Ziemons <benedikt.ziemons@cern.ch>, 2020-2021
 # - Radu Carpa <radu.carpa@cern.ch>, 2021
-# - Mayank Sharma <mayank.sharma@cern.ch>, 2021
+# - Mayank Sharma <imptodefeat@gmail.com>, 2021
 # - Simon Fayer <simon.fayer05@imperial.ac.uk>, 2021
 # - Rakshita Varadarajan <rakshitajps@gmail.com>, 2021
-# - Cedric Serfon <cedric.serfon@cern.ch>, 2021
+# - Mario Lassnig <mario.lassnig@cern.ch>, 2021
+# - Cedric Serfon <cedric.serfon@cern.ch>, 2021-2022
 
 from __future__ import print_function
 
@@ -138,6 +139,13 @@ def root_account(vo):
     return InternalAccount('root', vo=vo)
 
 
+@pytest.fixture(scope='module')
+def jdoe_account(vo):
+    from rucio.common.types import InternalAccount
+
+    return InternalAccount('jdoe', vo=vo)
+
+
 @pytest.fixture(scope="module")
 def containerized_rses(rucio_client):
     """
diff --git a/lib/rucio/tests/test_bb8.py b/lib/rucio/tests/test_bb8.py
--- a/lib/rucio/tests/test_bb8.py
+++ b/lib/rucio/tests/test_bb8.py
@@ -1,5 +1,5 @@
 # -*- coding: utf-8 -*-
-# Copyright 2016-2021 CERN
+# Copyright 2016-2022 CERN
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -22,107 +22,259 @@
 # - Benedikt Ziemons <benedikt.ziemons@cern.ch>, 2020-2021
 # - Mario Lassnig <mario.lassnig@cern.ch>, 2020
 # - Simon Fayer <simon.fayer05@imperial.ac.uk>, 2021
+# - Cedric Serfon <cedric.serfon@cern.ch>, 2021-2022
 
-import unittest
-from datetime import datetime
+from datetime import datetime, timedelta
 
 import pytest
 
-from rucio.common.config import config_get_bool
 from rucio.common.exception import RuleNotFound, UnsupportedOperation
-from rucio.common.types import InternalAccount, InternalScope
-from rucio.common.utils import generate_uuid as uuid
 from rucio.core.account_limit import set_local_account_limit
-from rucio.core.did import add_did, attach_dids
+from rucio.core import config as config_core
+from rucio.core.did import attach_dids, set_status, set_metadata
 from rucio.core.lock import successful_transfer
-from rucio.core.rse import add_rse_attribute, get_rse_id
-from rucio.core.rule import add_rule, get_rule, delete_rule
+from rucio.core.replica import add_replicas
+from rucio.core.rse import add_rse_attribute, fill_rse_expired, get_rse_usage, set_rse_usage
+from rucio.core.rule import add_rule, get_rule, delete_rule, update_rule
+from rucio.core.rse_expression_parser import REGION
+from rucio.daemons.abacus.rse import run as run_abacus
 from rucio.daemons.bb8.common import rebalance_rule
+from rucio.daemons.bb8.bb8 import run as bb8_run
 from rucio.daemons.judge.cleaner import rule_cleaner
-from rucio.db.sqla.constants import DIDType, RuleState
-from rucio.tests.common_server import get_vo
+from rucio.daemons.judge.evaluator import re_evaluator
+from rucio.daemons.undertaker import undertaker
+from rucio.db.sqla.constants import RuleState
 from rucio.tests.test_rule import create_files, tag_generator
 
 
-@pytest.mark.noparallel(reason='uses daemon and pre-defined rses')
-class TestJudgeCleaner(unittest.TestCase):
+@pytest.mark.noparallel(reason='uses daemons')
+def test_bb8_rebalance_rule(vo, root_account, jdoe_account, rse_factory, mock_scope, did_factory):
+    """BB8: Test the rebalance rule method"""
+    rse1, rse1_id = rse_factory.make_posix_rse()
+    rse2, rse2_id = rse_factory.make_posix_rse()
 
-    @classmethod
-    def setUpClass(cls):
-        if config_get_bool('common', 'multi_vo', raise_exception=False, default=False):
-            cls.vo = {'vo': get_vo()}
+    # Add Tags
+    T1 = tag_generator()
+    T2 = tag_generator()
+    add_rse_attribute(rse1_id, T1, True)
+    add_rse_attribute(rse2_id, T2, True)
+
+    # Add fake weights
+    add_rse_attribute(rse1_id, "fakeweight", 10)
+    add_rse_attribute(rse2_id, "fakeweight", 0)
+
+    # Add quota
+    set_local_account_limit(jdoe_account, rse1_id, -1)
+    set_local_account_limit(jdoe_account, rse2_id, -1)
+
+    set_local_account_limit(root_account, rse1_id, -1)
+    set_local_account_limit(root_account, rse2_id, -1)
+
+    files = create_files(3, mock_scope, rse1_id)
+    dataset = did_factory.make_dataset()
+    attach_dids(mock_scope, dataset['name'], files, jdoe_account)
+    set_status(mock_scope, dataset['name'], open=False)
+
+    # Invalid the cache because the result of parse_expression is cached
+    REGION.invalidate()
+
+    rule_id = add_rule(dids=[{'scope': mock_scope, 'name': dataset['name']}], account=jdoe_account, copies=1, rse_expression=rse1, grouping='NONE', weight='fakeweight', lifetime=None, locked=False, subscription_id=None)[0]
+    rule = {}
+    try:
+        rule = get_rule(rule_id)
+    except:
+        pytest.raises(RuleNotFound, get_rule, rule_id)
+    child_rule = rebalance_rule(rule, 'Rebalance', rse2, priority=3)
+
+    rule_cleaner(once=True)
+
+    assert(get_rule(rule_id)['expires_at'] <= datetime.utcnow())
+    assert(get_rule(rule_id)['child_rule_id'] == child_rule)
+
+    rule_cleaner(once=True)
+
+    assert(get_rule(rule_id)['expires_at'] <= datetime.utcnow())
+
+    successful_transfer(scope=mock_scope, name=files[0]['name'], rse_id=rse2_id, nowait=False)
+    successful_transfer(scope=mock_scope, name=files[1]['name'], rse_id=rse2_id, nowait=False)
+    with pytest.raises(UnsupportedOperation):
+        delete_rule(rule_id)
+    successful_transfer(scope=mock_scope, name=files[2]['name'], rse_id=rse2_id, nowait=False)
+
+    rule_cleaner(once=True)
+    assert(get_rule(child_rule)['state'] == RuleState.OK)
+    set_metadata(mock_scope, dataset['name'], 'lifetime', -86400)
+    undertaker.run(once=True)
+
+
+@pytest.mark.noparallel(reason='uses daemons')
+def test_bb8_full_workflow(vo, root_account, jdoe_account, rse_factory, mock_scope, did_factory):
+    """BB8: Test the rebalance rule method"""
+    config_core.set(section='bb8', option='allowed_accounts', value='jdoe')
+    tot_rses = 4
+    rses = [rse_factory.make_posix_rse() for _ in range(tot_rses)]
+    rse1, rse1_id = rses[0]
+    rse2, rse2_id = rses[1]
+    rse3, rse3_id = rses[2]
+    rse4, rse4_id = rses[3]
+
+    # Add Tags
+    # RSE 1 and 2 nmatch expression T1=true
+    # RSE 3 and 4 nmatch expression T2=true
+    T1 = tag_generator()
+    T2 = tag_generator()
+    add_rse_attribute(rse1_id, T1, True)
+    add_rse_attribute(rse2_id, T1, True)
+    add_rse_attribute(rse3_id, T2, True)
+    add_rse_attribute(rse4_id, T2, True)
+
+    # Add fake weights
+    add_rse_attribute(rse1_id, "fakeweight", 10)
+    add_rse_attribute(rse2_id, "fakeweight", 0)
+    add_rse_attribute(rse3_id, "fakeweight", 0)
+    add_rse_attribute(rse4_id, "fakeweight", 0)
+    add_rse_attribute(rse1_id, "freespace", 1)
+    add_rse_attribute(rse2_id, "freespace", 1)
+    add_rse_attribute(rse3_id, "freespace", 1)
+    add_rse_attribute(rse4_id, "freespace", 1)
+
+    # Add quota
+    set_local_account_limit(jdoe_account, rse1_id, -1)
+    set_local_account_limit(jdoe_account, rse2_id, -1)
+    set_local_account_limit(jdoe_account, rse3_id, -1)
+    set_local_account_limit(jdoe_account, rse4_id, -1)
+
+    set_local_account_limit(root_account, rse1_id, -1)
+    set_local_account_limit(root_account, rse2_id, -1)
+    set_local_account_limit(root_account, rse3_id, -1)
+    set_local_account_limit(root_account, rse4_id, -1)
+
+    # Invalid the cache because the result of parse_expression is cached
+    REGION.invalidate()
+
+    tot_datasets = 4
+    # Create a list of datasets
+    datasets = [did_factory.make_dataset() for _ in range(tot_datasets)]
+    dsn = [dataset['name'] for dataset in datasets]
+
+    rules = list()
+
+    base_unit = 100000000000
+    nb_files1 = 7
+    nb_files2 = 5
+    nb_files3 = 3
+    nb_files4 = 2
+    file_size = 1 * base_unit
+    rule_to_rebalance = None
+
+    # Add one secondary file
+    files = create_files(1, mock_scope, rse1_id, bytes_=1)
+    add_rule(dids=[{'scope': mock_scope, 'name': files[0]['name']}], account=jdoe_account, copies=1, rse_expression=rse1, grouping='DATASET', weight=None, lifetime=-86400, locked=False, subscription_id=None)[0]
+    for cnt in range(3, tot_rses):
+        add_replicas(rses[cnt][1], files, jdoe_account)
+        add_rule(dids=[{'scope': mock_scope, 'name': files[0]['name']}], account=jdoe_account, copies=1, rse_expression=rses[cnt][0], grouping='DATASET', weight=None, lifetime=-86400, locked=False, subscription_id=None)[0]
+    rule_cleaner(once=True)
+
+    # Create dataset 1 of 800 GB and create a rule on RSE 1 and RSE 3
+    files = create_files(nb_files1, mock_scope, rse1_id, bytes_=file_size)
+    attach_dids(mock_scope, dsn[0], files, jdoe_account)
+
+    rule_id = add_rule(dids=[{'scope': mock_scope, 'name': dsn[0]}], account=jdoe_account, copies=1, rse_expression=rse1, grouping='DATASET', weight=None, lifetime=None, locked=False, subscription_id=None)[0]
+    rules.append(rule_id)
+
+    add_replicas(rse3_id, files, jdoe_account)
+    rule_id = add_rule(dids=[{'scope': mock_scope, 'name': dsn[0]}], account=jdoe_account, copies=1, rse_expression=rse3, grouping='DATASET', weight=None, lifetime=None, locked=False, subscription_id=None)[0]
+    rules.append(rule_id)
+
+    # Create dataset 2 of 500 GB and create a rule on RSE 1 and RSE 2
+    files = create_files(nb_files2, mock_scope, rse1_id, bytes_=file_size)
+    attach_dids(mock_scope, dsn[1], files, jdoe_account)
+
+    rule_id = add_rule(dids=[{'scope': mock_scope, 'name': dsn[1]}], account=jdoe_account, copies=1, rse_expression=rse1, grouping='DATASET', weight=None, lifetime=None, locked=False, subscription_id=None)[0]
+    rules.append(rule_id)
+
+    add_replicas(rse2_id, files, jdoe_account)
+    rule_id = add_rule(dids=[{'scope': mock_scope, 'name': dsn[1]}], account=jdoe_account, copies=1, rse_expression=rse2, grouping='DATASET', weight=None, lifetime=None, locked=False, subscription_id=None)[0]
+    rules.append(rule_id)
+
+    # Create dataset 3 of 300 GB and create a rule on RSE 1. The copy on RSE 3 is secondary
+    files = create_files(nb_files3, mock_scope, rse1_id, bytes_=file_size)
+    attach_dids(mock_scope, dsn[2], files, jdoe_account)
+
+    rule_id = add_rule(dids=[{'scope': mock_scope, 'name': dsn[2]}], account=jdoe_account, copies=1, rse_expression=rse1, grouping='DATASET', weight=None, lifetime=None, locked=False, subscription_id=None)[0]
+    rule_to_rebalance = rule_id
+    rules.append(rule_id)
+
+    add_replicas(rse3_id, files, jdoe_account)
+    rule_id = add_rule(dids=[{'scope': mock_scope, 'name': dsn[2]}], account=jdoe_account, copies=1, rse_expression=rse3, grouping='DATASET', weight=None, lifetime=-86400, locked=False, subscription_id=None)[0]
+    rule_cleaner(once=True)
+    try:
+        rule = get_rule(rule_id)
+    except:
+        pytest.raises(RuleNotFound, get_rule, rule_id)
+
+    # Create dataset 4 of 200 GB and create a rule on RSE 3. The copy on RSE 2 is secondary
+    files = create_files(nb_files4, mock_scope, rse3_id, bytes_=file_size)
+    attach_dids(mock_scope, dsn[3], files, jdoe_account)
+
+    rule_id = add_rule(dids=[{'scope': mock_scope, 'name': dsn[3]}], account=jdoe_account, copies=1, rse_expression=rse3, grouping='DATASET', weight=None, lifetime=None, locked=False, subscription_id=None)[0]
+    rules.append(rule_id)
+
+    add_replicas(rse2_id, files, jdoe_account)
+    rule_id = add_rule(dids=[{'scope': mock_scope, 'name': dsn[3]}], account=jdoe_account, copies=1, rse_expression=rse2, grouping='DATASET', weight=None, lifetime=-86400, locked=False, subscription_id=None)[0]
+    rule_cleaner(once=True)
+    try:
+        rule = get_rule(rule_id)
+    except:
+        pytest.raises(RuleNotFound, get_rule, rule_id)
+
+    for dataset in dsn:
+        set_status(mock_scope, dataset, open=False)
+
+    for rse in rses:
+        fill_rse_expired(rse[1])
+        set_rse_usage(rse_id=rse[1], source='min_free_space', used=2 * base_unit, free=2 * base_unit, session=None)
+        set_rse_usage(rse_id=rse[1], source='storage', used=15 * base_unit, free=2 * base_unit, session=None)
+    set_rse_usage(rse_id=rse2_id, source='min_free_space', used=1 * base_unit, free=1 * base_unit, session=None)
+    set_rse_usage(rse_id=rse2_id, source='storage', used=6 * base_unit, free=5 * base_unit, session=None)
+
+    run_abacus(once=True, threads=1, fill_history_table=False, sleep_time=10)
+    # Summary :
+    # RSE 1 : 1500 GB primary + 1 B secondary
+    tot_space = [src for src in get_rse_usage(rse1_id) if src['source'] == 'rucio'][0]
+    expired = [src for src in get_rse_usage(rse1_id) if src['source'] == 'expired'][0]
+    assert tot_space['used'] == (nb_files1 + nb_files2 + nb_files3) * file_size + 1
+    assert expired['used'] == 1
+    # RSE 2 : 500 GB primary + 100 GB secondary
+    tot_space = [src for src in get_rse_usage(rse2_id) if src['source'] == 'rucio'][0]
+    expired = [src for src in get_rse_usage(rse2_id) if src['source'] == 'expired'][0]
+    assert tot_space['used'] == (nb_files2 + nb_files4) * file_size
+    assert expired['used'] == nb_files4 * file_size
+    # Total primary on T1=true : 2000 GB
+    # Total secondary on T1=true : 200 GB
+    # Ratio secondary / primary = 10  %
+    # Ratio on RSE 1 : 0 %
+    # Ratio on RSE 2 : 40 %
+
+    # Now run BB8
+
+    re_evaluator(once=True, sleep_time=30, did_limit=100)
+    bb8_run(once=True, rse_expression='%s=true' % str(T1), move_subscriptions=False, use_dump=False, sleep_time=300, threads=1, dry_run=False)
+
+    for rule_id in rules:
+        rule = get_rule(rule_id)
+        if rule_id != rule_to_rebalance:
+            assert(rule['child_rule_id'] is None)
         else:
-            cls.vo = {}
-
-        # Add test RSE
-        cls.rse1 = 'MOCK'
-        cls.rse3 = 'MOCK3'
-        cls.rse4 = 'MOCK4'
-        cls.rse5 = 'MOCK5'
-
-        cls.rse1_id = get_rse_id(cls.rse1, **cls.vo)
-        cls.rse3_id = get_rse_id(cls.rse3, **cls.vo)
-        cls.rse4_id = get_rse_id(cls.rse4, **cls.vo)
-        cls.rse5_id = get_rse_id(cls.rse5, **cls.vo)
-
-        # Add Tags
-        cls.T1 = tag_generator()
-        cls.T2 = tag_generator()
-        add_rse_attribute(cls.rse1_id, cls.T1, True)
-        add_rse_attribute(cls.rse3_id, cls.T1, True)
-        add_rse_attribute(cls.rse4_id, cls.T2, True)
-        add_rse_attribute(cls.rse5_id, cls.T1, True)
-
-        # Add fake weights
-        add_rse_attribute(cls.rse1_id, "fakeweight", 10)
-        add_rse_attribute(cls.rse3_id, "fakeweight", 0)
-        add_rse_attribute(cls.rse4_id, "fakeweight", 0)
-        add_rse_attribute(cls.rse5_id, "fakeweight", 0)
-
-        # Add quota
-        cls.jdoe = InternalAccount('jdoe', **cls.vo)
-        cls.root = InternalAccount('root', **cls.vo)
-        set_local_account_limit(cls.jdoe, cls.rse1_id, -1)
-        set_local_account_limit(cls.jdoe, cls.rse3_id, -1)
-        set_local_account_limit(cls.jdoe, cls.rse4_id, -1)
-        set_local_account_limit(cls.jdoe, cls.rse5_id, -1)
-
-        set_local_account_limit(cls.root, cls.rse1_id, -1)
-        set_local_account_limit(cls.root, cls.rse3_id, -1)
-        set_local_account_limit(cls.root, cls.rse4_id, -1)
-        set_local_account_limit(cls.root, cls.rse5_id, -1)
-
-    def test_bb8_rebalance_rule(self):
-        """ BB8: Test the rebalance rule method"""
-        scope = InternalScope('mock', **self.vo)
-        files = create_files(3, scope, self.rse1_id)
-        dataset = 'dataset_' + str(uuid())
-        add_did(scope, dataset, DIDType.DATASET, self.jdoe)
-        attach_dids(scope, dataset, files, self.jdoe)
-
-        rule_id = add_rule(dids=[{'scope': scope, 'name': dataset}], account=self.jdoe, copies=1, rse_expression=self.rse1, grouping='NONE', weight='fakeweight', lifetime=None, locked=False, subscription_id=None)[0]
-
-        rule = {}
-        try:
-            rule = get_rule(rule_id)
-        except:
-            pytest.raises(RuleNotFound, get_rule, rule_id)
-        child_rule = rebalance_rule(rule, 'Rebalance', self.rse3, priority=3)
-
-        rule_cleaner(once=True)
-
-        assert(get_rule(rule_id)['expires_at'] <= datetime.utcnow())
-        assert(get_rule(rule_id)['child_rule_id'] == child_rule)
-
-        rule_cleaner(once=True)
-
-        assert(get_rule(rule_id)['expires_at'] <= datetime.utcnow())
-
-        successful_transfer(scope=scope, name=files[0]['name'], rse_id=self.rse3_id, nowait=False)
-        successful_transfer(scope=scope, name=files[1]['name'], rse_id=self.rse3_id, nowait=False)
-        with pytest.raises(UnsupportedOperation):
-            delete_rule(rule_id)
-        successful_transfer(scope=scope, name=files[2]['name'], rse_id=self.rse3_id, nowait=False)
-
-        rule_cleaner(once=True)
-        assert(get_rule(child_rule)['state'] == RuleState.OK)
+            assert(rule['child_rule_id'] is not None)
+            assert(rule['expires_at'] <= datetime.utcnow() + timedelta(seconds=1))  # timedelta needed to prevent failure due to rounding effects
+            child_rule_id = rule['child_rule_id']
+            child_rule = get_rule(child_rule_id)
+            assert(child_rule['rse_expression'] == rse2)
+            # For teardown, delete child rule
+            update_rule(child_rule_id, {'lifetime': -86400})
+    rule_cleaner(once=True)
+
+    for dataset in dsn:
+        set_metadata(mock_scope, dataset, 'lifetime', -86400)
+    undertaker.run(once=True)
diff --git a/lib/rucio/tests/test_did.py b/lib/rucio/tests/test_did.py
--- a/lib/rucio/tests/test_did.py
+++ b/lib/rucio/tests/test_did.py
@@ -1,5 +1,5 @@
 # -*- coding: utf-8 -*-
-# Copyright 2013-2021 CERN
+# Copyright 2013-2022 CERN
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -17,7 +17,7 @@
 # - Vincent Garonne <vincent.garonne@cern.ch>, 2013-2018
 # - Martin Barisits <martin.barisits@cern.ch>, 2013-2020
 # - Mario Lassnig <mario.lassnig@cern.ch>, 2013-2020
-# - Cedric Serfon <cedric.serfon@cern.ch>, 2013-2021
+# - Cedric Serfon <cedric.serfon@cern.ch>, 2013-2022
 # - Ralph Vigne <ralph.vigne@cern.ch>, 2013
 # - Yun-Pin Sun <winter0128@gmail.com>, 2013
 # - Thomas Beermann <thomas.beermann@cern.ch>, 2013-2018
@@ -30,6 +30,8 @@
 # - Jaroslav Guenther <jaroslav.guenther@cern.ch>, 2020
 # - Benedikt Ziemons <benedikt.ziemons@cern.ch>, 2020-2021
 # - Simon Fayer <simon.fayer05@imperial.ac.uk>, 2021
+# - David Población Criado <david.poblacion.criado@cern.ch>, 2021
+# - Rob Barnsley <robbarnsley@users.noreply.github.com>, 2021
 
 from __future__ import print_function
 
@@ -57,7 +59,7 @@
 from rucio.core.account_limit import set_local_account_limit
 from rucio.core.did import (list_dids, add_did, delete_dids, get_did_atime, touch_dids, attach_dids, detach_dids,
                             get_metadata, set_metadata, get_did, get_did_access_cnt, add_did_to_followed,
-                            get_users_following_did, remove_did_from_followed)
+                            get_users_following_did, remove_did_from_followed, set_status)
 from rucio.core.replica import add_replica
 from rucio.core.rse import get_rse_id
 from rucio.db.sqla.constants import DIDType
@@ -405,33 +407,6 @@ def test_list_recursive(self):
         assert tmp_dataset_2 not in dids
         assert len(dids) == 2
 
-    @pytest.mark.dirty
-    @pytest.mark.noparallel(reason='uses pre-defined scope, fails when run in parallel')
-    def test_list_by_length(self):
-        """ DATA IDENTIFIERS (CLIENT): List did with length """
-        tmp_scope = 'mock'
-
-        tmp_dsn = 'dsn_%s' % generate_uuid()
-        self.did_client.add_dataset(scope=tmp_scope, name=tmp_dsn)
-
-        dids = self.did_client.list_dids(tmp_scope, {'length.gt': 0})
-        results = []
-        for d in dids:
-            results.append(d)
-        assert len(results) != 0
-
-        dids = self.did_client.list_dids(tmp_scope, {'length.gt': -1, 'length.lt': 1})
-        results = []
-        for d in dids:
-            results.append(d)
-        assert len(results) == 0
-
-        dids = self.did_client.list_dids(tmp_scope, {'length': 0})
-        results = []
-        for d in dids:
-            results.append(d)
-        assert len(results) == 0
-
     @pytest.mark.dirty
     def test_list_by_metadata(self):
         """ DATA IDENTIFIERS (CLIENT): List did with metadata"""
@@ -1179,3 +1154,34 @@ def test_bulk_get_meta_inheritance(vo, rse_factory, mock_scope, did_factory, ruc
         met = meta_mapping[did]
         for key in met:
             assert met[key] == meta[key]
+
+
+@pytest.mark.dirty
+@pytest.mark.noparallel(reason='uses pre-defined scope')
+def test_list_by_length(vo, root_account, rse_factory, mock_scope, did_factory, did_client):
+    """ DATA IDENTIFIERS (CLIENT): List did with length """
+
+    tmp_scope = scope_name_generator()
+    scope.add_scope(tmp_scope, 'root', 'root', vo)
+    rse, rse_id = rse_factory.make_posix_rse()
+    dataset = did_factory.upload_test_dataset(rse, tmp_scope)
+    set_status(InternalScope(tmp_scope, vo), dataset[0]['dataset_name'], open=False)
+
+    dids = did_client.list_dids(tmp_scope, {'length.gt': 0})
+    results = []
+    for d in dids:
+        results.append(d)
+    print(results)
+    assert len(results) != 0
+
+    dids = did_client.list_dids(tmp_scope, {'length.gt': -1, 'length.lt': 1})
+    results = []
+    for d in dids:
+        results.append(d)
+    assert len(results) == 0
+
+    dids = did_client.list_dids(tmp_scope, {'length': 0})
+    results = []
+    for d in dids:
+        results.append(d)
+    assert len(results) == 0