Repository: rucio/rucio

Issue:
Reevaluate the list of RSEs to be processed by a reaper
Motivation
----------
The list of RSEs to be processed by a reaper instance is evaluated once at the beginning.


Modification
------------
Reevaluate the list of RSEs regularly




Discussion:

Resolving patch:
diff --git a/lib/rucio/core/rse_expression_parser.py b/lib/rucio/core/rse_expression_parser.py
--- a/lib/rucio/core/rse_expression_parser.py
+++ b/lib/rucio/core/rse_expression_parser.py
@@ -9,7 +9,7 @@
   - Martin Barisits, <martin.barisits@cern.ch>, 2013-2017
   - Vincent Garonne, <vincent.garonne@cern.ch>, 2013
   - Mario Lassnig, <mario.lassnig@cern.ch>, 2013
-  - Cedric Serfon, <cedric.serfon@cern.ch>, 2014
+  - Cedric Serfon, <cedric.serfon@cern.ch>, 2014-2019
   - Hannes Hansen, <hannes.jakob.hansen@cern.ch>, 2018-2019
   - Brandon White, <bjwhite@fnal.gov>, 2019
 
@@ -42,7 +42,7 @@
 
 
 REGION = make_region().configure('dogpile.cache.memcached',
-                                 expiration_time=3600,
+                                 expiration_time=600,
                                  arguments={'url': config_get('cache', 'url', False, '127.0.0.1:11211'), 'distributed_lock': True})
 
 
diff --git a/lib/rucio/daemons/reaper/reaper2.py b/lib/rucio/daemons/reaper/reaper2.py
--- a/lib/rucio/daemons/reaper/reaper2.py
+++ b/lib/rucio/daemons/reaper/reaper2.py
@@ -80,6 +80,44 @@
                                             'distributed_lock': True})
 
 
+def get_rses_to_process(rses, include_rses, exclude_rses):
+    """
+    Return the list of RSEs to process based on rses, include_rses and exclude_rses
+
+    :param rses:               List of RSEs the reaper should work against. If empty, it considers all RSEs.
+    :param exclude_rses:       RSE expression to exclude RSEs from the Reaper.
+    :param include_rses:       RSE expression to include RSEs.
+
+    :returns: A list of RSEs to process
+    """
+    result = REGION.get('rses_to_process')
+    if result is not NO_VALUE:
+        return result
+
+    all_rses = list_rses()
+    if rses:
+        invalid = set(rses) - set([rse['rse'] for rse in all_rses])
+        if invalid:
+            msg = 'RSE{} {} cannot be found'.format('s' if len(invalid) > 1 else '',
+                                                    ', '.join([repr(rse) for rse in invalid]))
+            raise RSENotFound(msg)
+        rses = [rse for rse in all_rses if rse['rse'] in rses]
+    else:
+        rses = all_rses
+
+    if include_rses:
+        included_rses = parse_expression(include_rses)
+        rses = [rse for rse in rses if rse in included_rses]
+
+    if exclude_rses:
+        excluded_rses = parse_expression(exclude_rses)
+        rses = [rse for rse in rses if rse not in excluded_rses]
+
+    REGION.set('rses_to_process', rses)
+    logging.info('Reaper: This instance will work on RSEs: %s', ', '.join([rse['rse'] for rse in rses]))
+    return rses
+
+
 def delete_from_storage(replicas, prot, rse_info, staging_areas, prepend_str):
     deleted_files = []
     rse_name = rse_info['rse']
@@ -182,6 +220,28 @@ def get_rses_to_hostname_mapping():
     return result
 
 
+def get_max_deletion_threads_by_hostname(hostname):
+    """
+    Internal method to check RSE usage and limits.
+
+    :param hostname: the hostname of the SE
+
+    :returns : The maximum deletion thread for the SE.
+    """
+    result = REGION.get('max_deletion_threads_%s' % hostname)
+    if result is NO_VALUE:
+        try:
+            max_deletion_thread = get('reaper', 'max_deletion_threads_%s' % hostname)
+        except ConfigNotFound:
+            try:
+                max_deletion_thread = get('reaper', 'nb_workers_by_hostname')
+            except ConfigNotFound:
+                max_deletion_thread = 5
+        REGION.set('max_deletion_threads_%s' % hostname, max_deletion_thread)
+        result = max_deletion_thread
+    return result
+
+
 def __check_rse_usage(rse, rse_id, prepend_str):
     """
     Internal method to check RSE usage and limits.
@@ -270,12 +330,14 @@ def __check_rse_usage(rse, rse_id, prepend_str):
     return result
 
 
-def reaper(rses, chunk_size=100, once=False, greedy=False,
+def reaper(rses, include_rses, exclude_rses, chunk_size=100, once=False, greedy=False,
            scheme=None, delay_seconds=0, sleep_time=60):
     """
     Main loop to select and delete files.
 
     :param rses:           List of RSEs the reaper should work against. If empty, it considers all RSEs.
+    :param exclude_rses:       RSE expression to exclude RSEs from the Reaper.
+    :param include_rses:       RSE expression to include RSEs.
     :param chunk_size:     The size of chunk for deletion.
     :param once:           If True, only runs one iteration of the main loop.
     :param greedy:         If True, delete right away replicas with tombstone.
@@ -283,11 +345,6 @@ def reaper(rses, chunk_size=100, once=False, greedy=False,
     :param delay_seconds:  The delay to query replicas in BEING_DELETED state.
     :param sleep_time:     Time between two cycles.
     """
-
-    try:
-        max_deletion_thread = get('reaper', 'nb_workers_by_hostname')
-    except ConfigNotFound as error:
-        max_deletion_thread = 5
     hostname = socket.getfqdn()
     executable = sys.argv[0]
     pid = os.getpid()
@@ -297,13 +354,18 @@ def reaper(rses, chunk_size=100, once=False, greedy=False,
     prepend_str = 'Thread [%i/%i] : ' % (heart_beat['assign_thread'] + 1, heart_beat['nr_threads'])
     logging.info('%s Reaper starting', prepend_str)
 
-    time.sleep(15)  # To prevent running on the same partition if all the reapers restart at the same time
+    time.sleep(10)  # To prevent running on the same partition if all the reapers restart at the same time
     heart_beat = live(executable, hostname, pid, hb_thread)
     prepend_str = 'Thread [%i/%i] : ' % (heart_beat['assign_thread'] + 1, heart_beat['nr_threads'])
     logging.info('%s Reaper started', prepend_str)
 
     while not GRACEFUL_STOP.is_set():
 
+        rses_to_process = get_rses_to_process(rses, include_rses, exclude_rses)
+        if not rses_to_process:
+            logging.error('%s Reaper: No RSEs found. Will sleep for 30 seconds', prepend_str)
+            time.sleep(30)
+            continue
         start_time = time.time()
         try:
             staging_areas = []
@@ -311,7 +373,7 @@ def reaper(rses, chunk_size=100, once=False, greedy=False,
             heart_beat = live(executable, hostname, pid, hb_thread, older_than=3600)
             prepend_str = 'Thread [%i/%i] : ' % (heart_beat['assign_thread'] + 1, heart_beat['nr_threads'])
             tot_needed_free_space = 0
-            for rse in rses:
+            for rse in rses_to_process:
                 # Check if the RSE is a staging area
                 if rse['staging_area']:
                     staging_areas.append(rse['rse'])
@@ -350,9 +412,10 @@ def reaper(rses, chunk_size=100, once=False, greedy=False,
                 list_rses_mult.extend([(rse_name, rse_id, dict_rses[rse_key][0], dict_rses[rse_key][1]) for _ in range(int(max_workers))])
             random.shuffle(list_rses_mult)
 
-            skip_until_next_run = []
             for rse_name, rse_id, needed_free_space, max_being_deleted_files in list_rses_mult:
-                if rse_id in skip_until_next_run:
+                result = REGION.get('pause_deletion_%s' % rse_id, expiration_time=120)
+                if result is not NO_VALUE:
+                    logging.info('%s Not enough replicas to delete on %s during the previous cycle. Deletion paused for a while', prepend_str, rse_name)
                     continue
                 logging.debug('%s Working on %s. Percentage of the total space needed %.2f', prepend_str, rse_name, needed_free_space / tot_needed_free_space * 100)
                 rse_hostname, rse_info = rses_hostname_mapping[rse_id]
@@ -368,6 +431,7 @@ def reaper(rses, chunk_size=100, once=False, greedy=False,
                         if key.split(',')[0] == str(rse_id):
                             tot_threads_for_rse += payload_cnt[key]
 
+                max_deletion_thread = get_max_deletion_threads_by_hostname(rse_hostname)
                 if rse_hostname_key in payload_cnt and tot_threads_for_hostname >= max_deletion_thread:
                     logging.debug('%s Too many deletion threads for %s on RSE %s. Back off', prepend_str, rse_hostname, rse_name)
                     # Might need to reschedule a try on this RSE later in the same cycle
@@ -388,7 +452,7 @@ def reaper(rses, chunk_size=100, once=False, greedy=False,
                     logging.debug('%s list_and_mark_unlocked_replicas  on %s for %s bytes in %s seconds: %s replicas', prepend_str, rse_name, needed_free_space, time.time() - del_start_time, len(replicas))
                     if len(replicas) < chunk_size:
                         logging.info('%s Not enough replicas to delete on %s (%s requested vs %s returned). Will skip any new attempts on this RSE until next cycle', prepend_str, rse_name, chunk_size, len(replicas))
-                        skip_until_next_run.append(rse_id)
+                        REGION.set('pause_deletion_%s' % rse_id, True)
 
                 except (DatabaseException, IntegrityError, DatabaseError) as error:
                     logging.error('%s %s', prepend_str, str(error))
@@ -475,35 +539,21 @@ def run(threads=1, chunk_size=100, once=False, greedy=False, rses=None, scheme=N
     """
     logging.info('main: starting processes')
 
-    all_rses = list_rses()
-
-    if rses:
-        invalid = set(rses) - set([rse['rse'] for rse in all_rses])
-        if invalid:
-            msg = 'RSE{} {} cannot be found'.format('s' if len(invalid) > 1 else '',
-                                                    ', '.join([repr(rse) for rse in invalid]))
-            raise RSENotFound(msg)
-        rses = [rse for rse in all_rses if rse['rse'] in rses]
-    else:
-        rses = all_rses
-
-    if include_rses:
-        included_rses = parse_expression(include_rses)
-        rses = [rse for rse in rses if rse in included_rses]
-
-    if exclude_rses:
-        excluded_rses = parse_expression(exclude_rses)
-        rses = [rse for rse in rses if rse not in excluded_rses]
-
-    if not rses:
+    rses_to_process = get_rses_to_process(rses, include_rses, exclude_rses)
+    if not rses_to_process:
         logging.error('Reaper: No RSEs found. Exiting.')
         return
 
-    logging.info('Reaper: This instance will work on RSEs: %s', ', '.join([rse['rse'] for rse in rses]))
+    logging.info('Reaper: This instance will work on RSEs: %s', ', '.join([rse['rse'] for rse in rses_to_process]))
+
+    # To populate the cache
+    get_rses_to_hostname_mapping()
 
     logging.info('starting reaper threads')
     threads_list = [threading.Thread(target=reaper, kwargs={'once': once,
                                                             'rses': rses,
+                                                            'include_rses': include_rses,
+                                                            'exclude_rses': exclude_rses,
                                                             'chunk_size': chunk_size,
                                                             'greedy': greedy,
                                                             'sleep_time': sleep_time,
@@ -515,9 +565,6 @@ def run(threads=1, chunk_size=100, once=False, greedy=False, rses=None, scheme=N
 
     logging.info('waiting for interrupts')
 
-    # To populate the cache
-    get_rses_to_hostname_mapping()
-
     # Interruptible joins require a timeout.
     while threads_list:
         threads_list = [thread.join(timeout=3.14) for thread in threads_list if thread and thread.isAlive()]


Test patch: