Repository: rucio/rucio

Issue:
dark-reaper not exposing the prometheus metrics port
Motivation
----------
The prometheus target is always down when enabling metrics via the helm chart. 


Modification
------------
Either fix the helm chart to not try to scrape dark reaper (difficult fix); or instantiate the prometheus metric endpoint (without any metrics) in dark-reaper (very simple fix) 



Discussion:
We selectively scrape things in CMS with pod annotations. 
Hi, thanks for the info. I didn't check the helm chart when writing this ticket and thought that pod annotations are common for all daemons. Still, even with them being per-component, I would like to avoid having to set them manually on each component just to avoid scrapping one component. So the ticket is still relevant IMHO

Resolving patch:
diff --git a/lib/rucio/daemons/reaper/dark_reaper.py b/lib/rucio/daemons/reaper/dark_reaper.py
--- a/lib/rucio/daemons/reaper/dark_reaper.py
+++ b/lib/rucio/daemons/reaper/dark_reaper.py
@@ -17,176 +17,169 @@
 Dark Reaper is a daemon to manage quarantined file deletion.
 '''
 
-import hashlib
+import functools
 import logging
-import os
 import random
-import socket
-import sys
 import threading
 import time
 import traceback
+from typing import TYPE_CHECKING
 
 import rucio.db.sqla.util
 from rucio.common import exception
 from rucio.common.config import config_get_bool
-from rucio.common.exception import (SourceNotFound, DatabaseException, ServiceUnavailable,
+from rucio.common.exception import (SourceNotFound, ServiceUnavailable,
                                     RSEAccessDenied, ResourceTemporaryUnavailable,
                                     RSENotFound, VONotFound)
-from rucio.common.logging import setup_logging, formatted_logger
-from rucio.common.utils import daemon_sleep
-from rucio.core.heartbeat import live, die, sanity_check
+from rucio.common.logging import setup_logging
 from rucio.core.message import add_message
+from rucio.core.monitor import record_counter
 from rucio.core.quarantined_replica import (list_quarantined_replicas,
                                             delete_quarantined_replicas,
                                             list_rses_with_quarantined_replicas)
 import rucio.core.rse as rse_core
 from rucio.core.rse_expression_parser import parse_expression
 from rucio.core.vo import list_vos
+from rucio.daemons.common import run_daemon
 from rucio.rse import rsemanager as rsemgr
 
+if TYPE_CHECKING:
+    from typing import Sequence, Optional
+    from rucio.daemons.common import HeartbeatHandler
+
 logging.getLogger("requests").setLevel(logging.CRITICAL)
 
 GRACEFUL_STOP = threading.Event()
 
 
-def reaper(rses, chunk_size=100, once=False, scheme=None, sleep_time=300):
+def reaper(
+        rses: "Sequence[str]",
+        chunk_size: int = 100,
+        once: bool = False,
+        scheme: "Optional[str]" = None,
+        sleep_time: int = 300,
+):
+    executable = 'dark-reaper'
+    if rses:
+        executable += ' --rses ' + ''.join(rses)
+    logger_prefix = 'dark-reaper'
+    run_daemon(
+        once=once,
+        graceful_stop=GRACEFUL_STOP,
+        executable=executable,
+        logger_prefix=logger_prefix,
+        partition_wait_time=10,
+        sleep_time=sleep_time,
+        run_once_fnc=functools.partial(
+            run_once,
+            rses=rses,
+            chunk_size=chunk_size,
+            scheme=scheme,
+        ),
+    )
+
+
+def run_once(
+        rses: "Sequence[str]",
+        heartbeat_handler: "HeartbeatHandler",
+        chunk_size: int = 100,
+        scheme: "Optional[str]" = None,
+        **_kwargs,
+):
     """
     Main loop to select and delete files.
 
     :param rses: List of RSEs the reaper should work against.
     :param chunk_size: the size of chunk for deletion.
-    :param once: If True, only runs one iteration of the main loop.
     :param scheme: Force the reaper to use a particular protocol, e.g., mock.
-    :param sleep_time: Thread sleep time after each chunk of work.
     """
-
-    pid = os.getpid()
-    thread = threading.current_thread()
-    hostname = socket.gethostname()
-    executable = ' '.join(sys.argv)
-    hash_executable = hashlib.sha256((sys.argv[0] + ''.join(rses)).encode()).hexdigest()
-    sanity_check(executable=None, hostname=hostname)
-
-    # heartbeat
-    heartbeat = live(executable=executable, hostname=hostname, pid=pid, thread=thread,
-                     hash_executable=hash_executable)
-    prepend_str = 'dark-reaper [%i/%i] : ' % (heartbeat['assign_thread'], heartbeat['nr_threads'])
-    logger = formatted_logger(logging.log, prepend_str + '%s')
+    worker_number, total_workers, logger = heartbeat_handler.live()
     logger(logging.INFO, 'Starting Dark Reaper on RSEs: %s', ', '.join(rses))
 
-    if not once:
-        logger(logging.INFO, 'Waiting for heartbeat synchonization')
-        GRACEFUL_STOP.wait(10)  # To prevent running on the same partition if all the reapers restart at the same time
+    nothing_to_do = True
+    rses_to_process = list(set(rses) & set(list_rses_with_quarantined_replicas()))
+    random.shuffle(rses_to_process)
+    for rse_id in rses_to_process:
+        worker_number, total_workers, logger = heartbeat_handler.live()
+        # The following query returns the list of real replicas (deleted_replicas) and list of dark replicas (dark_replicas)
+        # Real replicas can be directly removed from the quarantine table
+        deleted_replicas, dark_replicas = list_quarantined_replicas(rse_id=rse_id,
+                                                                    limit=chunk_size,
+                                                                    worker_number=worker_number,
+                                                                    total_workers=total_workers)
+
+        rse_info = rsemgr.get_rse_info(rse_id=rse_id)
+        rse = rse_info['rse']
+        prot = rsemgr.create_protocol(rse_info, 'delete', scheme=scheme)
 
-    while not GRACEFUL_STOP.is_set():
         try:
-            heartbeat = live(executable=executable, hostname=hostname, pid=pid, thread=thread,
-                             hash_executable=hash_executable)
-            prepend_str = 'dark-reaper [%i/%i] : ' % (heartbeat['assign_thread'], heartbeat['nr_threads'])
-            logger = formatted_logger(logging.log, prepend_str + '%s')
-            logger(logging.INFO, 'Live gives {0[heartbeat]}'.format(locals()))
-            nothing_to_do = True
-            start_time = time.time()
-
-            rses_to_process = list(set(rses) & set(list_rses_with_quarantined_replicas()))
-            random.shuffle(rses_to_process)
-            for rse_id in rses_to_process:
-                # The following query returns the list of real replicas (deleted_replicas) and list of dark replicas (dark_replicas)
-                # Real replicas can be directly removed from the quarantine table
-                deleted_replicas, dark_replicas = list_quarantined_replicas(rse_id=rse_id,
-                                                                            limit=chunk_size,
-                                                                            worker_number=heartbeat['assign_thread'],
-                                                                            total_workers=heartbeat['nr_threads'])
-
-                rse_info = rsemgr.get_rse_info(rse_id=rse_id)
-                rse = rse_info['rse']
-                prot = rsemgr.create_protocol(rse_info, 'delete', scheme=scheme)
-
-                heartbeat = live(executable=executable, hostname=hostname, pid=pid, thread=thread,
-                                 hash_executable=hash_executable)
-                prepend_str = 'dark-reaper [%i/%i] : ' % (heartbeat['assign_thread'], heartbeat['nr_threads'])
-                logger = formatted_logger(logging.log, prepend_str + '%s')
+            prot.connect()
+            for replica in dark_replicas:
+                worker_number, total_workers, logger = heartbeat_handler.live()
+                nothing_to_do = False
+                scope = ''
+                if replica['scope']:
+                    scope = replica['scope'].external
                 try:
-                    prot.connect()
-                    for replica in dark_replicas:
-                        nothing_to_do = False
-                        scope = ''
-                        if replica['scope']:
-                            scope = replica['scope'].external
-                        try:
-                            pfn = str(list(rsemgr.lfns2pfns(rse_settings=rse_info,
-                                                            lfns=[{'scope': scope,
-                                                                   'name': replica['name'],
-                                                                   'path': replica['path']}],
-                                                            operation='delete',
-                                                            scheme=scheme).values())[0])
-                            logger(logging.INFO, 'Deletion ATTEMPT of %s:%s as %s on %s', scope, replica['name'], pfn, rse)
-                            start = time.time()
-                            prot.delete(pfn)
-                            duration = time.time() - start
-                            logger(logging.INFO, 'Deletion SUCCESS of %s:%s as %s on %s in %s seconds', scope, replica['name'], pfn, rse, duration)
-                            payload = {'scope': scope,
-                                       'name': replica['name'],
-                                       'rse': rse,
-                                       'rse_id': rse_id,
-                                       'file-size': replica.get('bytes') or 0,
-                                       'bytes': replica.get('bytes') or 0,
-                                       'url': pfn,
-                                       'duration': duration,
-                                       'protocol': prot.attributes['scheme']}
-                            if replica['scope'].vo != 'def':
-                                payload['vo'] = replica['scope'].vo
-                            add_message('deletion-done', payload)
-                            deleted_replicas.append(replica)
-                        except SourceNotFound:
-                            err_msg = ('Deletion NOTFOUND of %s:%s as %s on %s'
-                                       % (scope, replica['name'], pfn, rse))
-                            logger(logging.WARNING, err_msg)
-                            deleted_replicas.append(replica)
-                        except (ServiceUnavailable, RSEAccessDenied, ResourceTemporaryUnavailable) as error:
-                            err_msg = ('Deletion NOACCESS of %s:%s as %s on %s: %s'
-                                       % (scope, replica['name'], pfn, rse, str(error)))
-                            logger(logging.WARNING, err_msg)
-                            payload = {'scope': scope,
-                                       'name': replica['name'],
-                                       'rse': rse,
-                                       'rse_id': rse_id,
-                                       'file-size': replica['bytes'] or 0,
-                                       'bytes': replica['bytes'] or 0,
-                                       'url': pfn,
-                                       'reason': str(error),
-                                       'protocol': prot.attributes['scheme']}
-                            if replica['scope'].vo != 'def':
-                                payload['vo'] = replica['scope'].vo
-                            add_message('deletion-failed', payload)
-
-                        except Exception:
-                            logging.critical(traceback.format_exc())
-                finally:
-                    prot.close()
-
-                delete_quarantined_replicas(rse_id=rse_id, replicas=deleted_replicas)
-
-                if once:
-                    break
-
-            if once:
-                break
-
-            if nothing_to_do:
-                logger(logging.INFO, 'Nothing to do')
-                daemon_sleep(start_time=start_time, sleep_time=sleep_time, graceful_stop=GRACEFUL_STOP)
-
-        except DatabaseException as error:
-            logging.warning('Reaper:  %s', str(error))
-        except Exception:
-            logging.critical(traceback.format_exc())
-
-    die(executable=executable, hostname=hostname, pid=pid, thread=thread, hash_executable=hash_executable)
-    logging.info('Graceful stop requested')
-    logging.info('Graceful stop done')
-    return
+                    pfn = str(list(rsemgr.lfns2pfns(rse_settings=rse_info,
+                                                    lfns=[{'scope': scope,
+                                                           'name': replica['name'],
+                                                           'path': replica['path']}],
+                                                    operation='delete',
+                                                    scheme=scheme).values())[0])
+                    logger(logging.INFO, 'Deletion ATTEMPT of %s:%s as %s on %s', scope, replica['name'], pfn, rse)
+                    start = time.time()
+                    prot.delete(pfn)
+                    record_counter('daemons.dark_reaper.deleted_replicas')
+                    duration = time.time() - start
+                    logger(logging.INFO, 'Deletion SUCCESS of %s:%s as %s on %s in %s seconds', scope, replica['name'], pfn, rse, duration)
+                    payload = {'scope': scope,
+                               'name': replica['name'],
+                               'rse': rse,
+                               'rse_id': rse_id,
+                               'file-size': replica.get('bytes') or 0,
+                               'bytes': replica.get('bytes') or 0,
+                               'url': pfn,
+                               'duration': duration,
+                               'protocol': prot.attributes['scheme']}
+                    if replica['scope'].vo != 'def':
+                        payload['vo'] = replica['scope'].vo
+                    add_message('deletion-done', payload)
+                    deleted_replicas.append(replica)
+                except SourceNotFound:
+                    err_msg = ('Deletion NOTFOUND of %s:%s as %s on %s'
+                               % (scope, replica['name'], pfn, rse))
+                    logger(logging.WARNING, err_msg)
+                    deleted_replicas.append(replica)
+                except (ServiceUnavailable, RSEAccessDenied, ResourceTemporaryUnavailable) as error:
+                    err_msg = ('Deletion NOACCESS of %s:%s as %s on %s: %s'
+                               % (scope, replica['name'], pfn, rse, str(error)))
+                    logger(logging.WARNING, err_msg)
+                    payload = {'scope': scope,
+                               'name': replica['name'],
+                               'rse': rse,
+                               'rse_id': rse_id,
+                               'file-size': replica['bytes'] or 0,
+                               'bytes': replica['bytes'] or 0,
+                               'url': pfn,
+                               'reason': str(error),
+                               'protocol': prot.attributes['scheme']}
+                    if replica['scope'].vo != 'def':
+                        payload['vo'] = replica['scope'].vo
+                    add_message('deletion-failed', payload)
+
+                except Exception:
+                    logging.critical(traceback.format_exc())
+        finally:
+            prot.close()
+
+        delete_quarantined_replicas(rse_id=rse_id, replicas=deleted_replicas)
+
+    must_sleep = False
+    if nothing_to_do:
+        logger(logging.INFO, 'Nothing to do')
+        must_sleep = True
+    return must_sleep
 
 
 def stop(signum=None, frame=None):


Test patch:
diff --git a/lib/rucio/tests/test_reaper.py b/lib/rucio/tests/test_reaper.py
--- a/lib/rucio/tests/test_reaper.py
+++ b/lib/rucio/tests/test_reaper.py
@@ -31,6 +31,7 @@
 from rucio.core import rse as rse_core
 from rucio.core import rule as rule_core
 from rucio.daemons.reaper.reaper import reaper
+from rucio.daemons.reaper.dark_reaper import reaper as dark_reaper
 from rucio.daemons.reaper.reaper import run as run_reaper
 from rucio.db.sqla.models import ConstituentAssociationHistory
 from rucio.db.sqla.session import read_session
@@ -106,6 +107,10 @@ def test_reaper(vo, caches_mock, file_config_mock, message_mock):
     assert len(msgs) == 50  # one for each deleted file
     assert all(msg['payload']['datatype'] == 'SOME_DATATYPE' for msg in msgs)
 
+    # run dark reaper just to catch the simplest possible errors in this daemon.
+    # TODO: remove this when/if we implement good testing for dark reaper
+    dark_reaper(once=True, rses=[rse_id])
+
 
 @pytest.mark.parametrize("file_config_mock", [
     # Run test twice: with, and without, temp tables