Repository: rucio/rucio

Issue:
bad activemq connection handling in kronos
`--sleep-time-files` in Kronos defines the interval at which to re-create a new activemq handler. But also the timeout of that connection. 

https://github.com/rucio/rucio/blob/b0f7064bc4b07f5be704d656bacdd96e7a03426b/lib/rucio/daemons/tracer/kronos.py#L377

As a consequence, Kronos is re-connecting to the brokers each `--sleep-time-files` seconds, while the currently active connections are force-closed, sometimes resulting in errors putting back the messages into the queue and most probably lost messages. 

This, obviously, is not the correct way of handing a connection. It's possible that the only problem is related to the value of timeout being too small


Discussion:

Resolving patch:
diff --git a/lib/rucio/common/stomp_utils.py b/lib/rucio/common/stomp_utils.py
--- a/lib/rucio/common/stomp_utils.py
+++ b/lib/rucio/common/stomp_utils.py
@@ -18,32 +18,143 @@
 """
 
 import socket
+from time import monotonic
+from typing import TYPE_CHECKING
 
 from stomp import Connection
 
 import logging
 
+if TYPE_CHECKING:
+    from typing import Any, Callable, List, Sequence, Tuple
 
-def get_stomp_brokers(brokers, port, use_ssl, vhost, reconnect_attempts, ssl_key_file, ssl_cert_file, timeout,
-                      logger=logging.log):
-    logger(logging.DEBUG, 'resolving broker dns alias: %s' % brokers)
+    LoggerFunction = Callable[..., Any]
 
-    brokers_resolved = []
-    for broker in brokers:
-        addrinfos = socket.getaddrinfo(broker, 0, socket.AF_INET, 0, socket.IPPROTO_TCP)
-        brokers_resolved.extend(ai[4][0] for ai in addrinfos)
 
-    logger(logging.DEBUG, 'broker resolved to %s', brokers_resolved)
-    conns = []
-    for broker in brokers_resolved:
-        conn = Connection(
-            host_and_ports=[(broker, port)],
-            vhost=vhost,
-            timeout=timeout,
+def resolve_ips(fqdns: "Sequence[str]", logger: "LoggerFunction" = logging.log):
+    logger(logging.DEBUG, 'resolving dns aliases: %s' % fqdns)
+    resolved = []
+    for fqdn in fqdns:
+        addrinfos = socket.getaddrinfo(fqdn, 0, socket.AF_INET, 0, socket.IPPROTO_TCP)
+        resolved.extend(ai[4][0] for ai in addrinfos)
+    logger(logging.DEBUG, 'dns aliases resolved to %s', resolved)
+    return resolved
+
+
+class StompConnectionManager:
+
+    def __init__(self):
+        self._brokers = None
+        self._port = None
+        self._use_ssl = None
+        self._vhost = None
+        self._reconnect_attempts = None
+        self._ssl_key_file = None
+        self._timeout = None
+        self._heartbeats = None
+
+        self._connections = {}
+
+    def is_stalled(self, connection: Connection, *, logger: "LoggerFunction" = logging.log):
+        if not connection.is_connected():
+            return True
+
+        if self._heartbeats and getattr(connection, 'received_heartbeat') and connection.received_heartbeat:
+            heartbeat_period_seconds = max(0, self._heartbeats[0], self._heartbeats[1]) / 1000
+
+            if not heartbeat_period_seconds:
+                return False
+
+            now = monotonic()
+            if connection.received_heartbeat + 10 * heartbeat_period_seconds < now:
+                logger(logging.WARNING, "Stomp connection missed heartbeats for a long time")
+                return True
+
+        return False
+
+    def disconnect(self):
+        for conn in self._connections.values():
+            if not conn.is_connected():
+                conn.disconnect()
+
+    def re_configure(
+            self,
+            brokers: "Sequence[str]",
+            port: int,
+            use_ssl: bool,
+            vhost,
+            reconnect_attempts: int,
+            ssl_key_file,
+            ssl_cert_file,
+            timeout,
             heartbeats=(0, 1000),
-            reconnect_attempts_max=reconnect_attempts
-        )
-        if use_ssl:
-            conn.set_ssl(key_file=ssl_key_file, cert_file=ssl_cert_file)
-        conns.append(conn)
-    return conns
+            *,
+            logger: "LoggerFunction" = logging.log
+    ) -> "Tuple[List, List]":
+
+        configuration_changed = any([
+            self._brokers != brokers,
+            self._port != port,
+            self._use_ssl != use_ssl,
+            self._vhost != vhost,
+            self._reconnect_attempts != reconnect_attempts,
+            self._ssl_key_file != ssl_key_file,
+            self._timeout != timeout,
+            self._heartbeats != heartbeats,
+        ])
+        if configuration_changed:
+            self._brokers = brokers
+            self._port = port
+            self._use_ssl = use_ssl
+            self._vhost = vhost
+            self._reconnect_attempts = reconnect_attempts
+            self._ssl_key_file = ssl_key_file
+            self._timeout = timeout
+            self._heartbeats = heartbeats
+
+        current_remotes = set(self._connections)
+        desired_remotes = set((ip, port) for ip in resolve_ips(brokers, logger=logger))
+
+        if configuration_changed:
+            # Re-create all connections
+            to_delete = current_remotes
+            to_create = desired_remotes
+        else:
+            to_delete = current_remotes.difference(desired_remotes)
+            to_create = desired_remotes.difference(current_remotes)
+
+            for remote in current_remotes.intersection(desired_remotes):
+                conn = self._connections[remote]
+
+                if self.is_stalled(conn, logger=logger):
+                    # Re-create stalled connections
+                    to_delete.add(remote)
+                    to_create.add(remote)
+
+        deleted_conns = []
+        for remote in to_delete:
+            conn = self._connections.pop(remote)
+            if conn.is_connected():
+                conn.disconnect()
+            deleted_conns.append(to_delete)
+
+        created_conns = []
+        for remote in to_create:
+            conn = Connection(
+                host_and_ports=[remote],
+                vhost=vhost,
+                timeout=timeout,
+                heartbeats=heartbeats,
+                reconnect_attempts_max=reconnect_attempts
+            )
+            if use_ssl:
+                conn.set_ssl(key_file=ssl_key_file, cert_file=ssl_cert_file)
+            self._connections[remote] = conn
+            created_conns.append(conn)
+
+        if not to_delete and not to_create:
+            logger(logging.INFO, "Stomp connections didn't change")
+        else:
+            logger(logging.INFO, f"Stomp connections refreshed. Deleted: {list(to_delete)}. Added: {list(to_create)}")
+
+        return created_conns, deleted_conns
diff --git a/lib/rucio/daemons/cache/consumer.py b/lib/rucio/daemons/cache/consumer.py
--- a/lib/rucio/daemons/cache/consumer.py
+++ b/lib/rucio/daemons/cache/consumer.py
@@ -25,9 +25,9 @@
 
 import rucio.db.sqla.util
 from rucio.common import exception
-from rucio.common.config import config_get, config_get_int, config_get_bool
+from rucio.common.config import config_get, config_get_int, config_get_bool, config_get_list
 from rucio.common.logging import setup_logging, formatted_logger
-from rucio.common.stomp_utils import get_stomp_brokers
+from rucio.common.stomp_utils import StompConnectionManager
 from rucio.common.types import InternalScope
 from rucio.core.monitor import MetricManager
 from rucio.core.rse import get_rse_id
@@ -105,17 +105,9 @@ def consumer(id_, num_thread=1):
 
     logger(logging.INFO, 'Rucio Cache consumer starting')
 
-    try:
-        brokers_alias = [b.strip() for b in config_get('messaging-cache', 'brokers').split(',')]
-    except:
-        raise Exception('Could not load rucio cache brokers from configuration')
-
-    use_ssl = True
-    try:
-        use_ssl = config_get_bool('messaging-cache', 'use_ssl')
-    except Exception:
-        pass
+    brokers = config_get_list('messaging-cache', 'brokers')
 
+    use_ssl = config_get_bool('messaging-cache', 'use_ssl', default=True, raise_exception=False)
     if not use_ssl:
         username = config_get('messaging-cache', 'username')
         password = config_get('messaging-cache', 'password')
@@ -128,15 +120,18 @@ def consumer(id_, num_thread=1):
     ssl_key_file = config_get('messaging-cache', 'ssl_key_file', raise_exception=False)
     ssl_cert_file = config_get('messaging-cache', 'ssl_cert_file', raise_exception=False)
 
-    conns = get_stomp_brokers(brokers=brokers_alias,
-                              port=port,
-                              use_ssl=use_ssl,
-                              vhost=vhost,
-                              reconnect_attempts=reconnect_attempts,
-                              ssl_key_file=ssl_key_file,
-                              ssl_cert_file=ssl_cert_file,
-                              timeout=None,
-                              logger=logger)
+    stomp_conn_mngr = StompConnectionManager()
+    conns, _ = stomp_conn_mngr.re_configure(
+        brokers=brokers,
+        port=port,
+        use_ssl=use_ssl,
+        vhost=vhost,
+        reconnect_attempts=reconnect_attempts,
+        ssl_key_file=ssl_key_file,
+        ssl_cert_file=ssl_cert_file,
+        timeout=None,
+        logger=logger
+    )
 
     logger(logging.INFO, 'consumer started')
 
@@ -157,13 +152,7 @@ def consumer(id_, num_thread=1):
         time.sleep(1)
 
     logger(logging.INFO, 'graceful stop requested')
-
-    for conn in conns:
-        try:
-            conn.disconnect()
-        except:
-            pass
-
+    stomp_conn_mngr.disconnect()
     logger(logging.INFO, 'graceful stop done')
 
 
diff --git a/lib/rucio/daemons/tracer/kronos.py b/lib/rucio/daemons/tracer/kronos.py
--- a/lib/rucio/daemons/tracer/kronos.py
+++ b/lib/rucio/daemons/tracer/kronos.py
@@ -25,14 +25,13 @@
 from json import loads as jloads, dumps as jdumps
 from threading import Event, Thread
 from time import time
-from typing import Dict
 
 import rucio.db.sqla.util
 from rucio.daemons.common import HeartbeatHandler, run_daemon
-from rucio.common.config import config_get, config_get_bool, config_get_int
+from rucio.common.config import config_get, config_get_bool, config_get_int, config_get_list
 from rucio.common.exception import RSENotFound, DatabaseException
 from rucio.common.logging import setup_logging
-from rucio.common.stomp_utils import get_stomp_brokers
+from rucio.common.stomp_utils import StompConnectionManager
 from rucio.common.stopwatch import Stopwatch
 from rucio.common.types import InternalAccount, InternalScope
 from rucio.core.did import touch_dids, list_parent_dids
@@ -51,7 +50,7 @@
 graceful_stop = Event()
 
 
-class AMQConsumer(object):
+class AMQConsumer:
     """ActiveMQ message consumer"""
 
     def __init__(self, broker, conn, queue, chunksize, subscription_id, excluded_usrdns, dataset_queue, bad_files_patterns, logger=logging.log):
@@ -298,7 +297,7 @@ def kronos_file(once: bool = False, dataset_queue: Queue = None, sleep_time: int
     """
     Main loop to consume tracer reports.
     """
-    return_values = {}
+    stomp_conn_mngr = StompConnectionManager()
     run_daemon(
         once=once,
         graceful_stop=graceful_stop,
@@ -308,19 +307,15 @@ def kronos_file(once: bool = False, dataset_queue: Queue = None, sleep_time: int
         sleep_time=sleep_time,
         run_once_fnc=functools.partial(
             run_once_kronos_file,
-            return_values=return_values,
+            stomp_conn_mngr=stomp_conn_mngr,
             dataset_queue=dataset_queue,
             sleep_time=sleep_time,
         )
     )
-    for conn in return_values['conns']:
-        try:
-            conn.disconnect()
-        except Exception:
-            pass
+    stomp_conn_mngr.disconnect()
 
 
-def run_once_kronos_file(heartbeat_handler: HeartbeatHandler, return_values: Dict, dataset_queue: Queue, sleep_time: int, **kwargs):
+def run_once_kronos_file(heartbeat_handler: HeartbeatHandler, stomp_conn_mngr: StompConnectionManager, dataset_queue: Queue, sleep_time: int, **kwargs):
     """
     Run the amq consumer once.
     """
@@ -342,10 +337,8 @@ def run_once_kronos_file(heartbeat_handler: HeartbeatHandler, return_values: Dic
     except Exception as error:
         logger.error(f'Failed to get bad_file_patterns {str(error)}')
         bad_files_patterns = []
-    try:
-        use_ssl = config_get_bool('tracer-kronos', 'use_ssl')
-    except Exception:
-        pass
+
+    use_ssl = config_get_bool('tracer-kronos', 'use_ssl', default=True, raise_exception=False)
     if not use_ssl:
         username = config_get('tracer-kronos', 'username')
         password = config_get('tracer-kronos', 'password')
@@ -353,12 +346,13 @@ def run_once_kronos_file(heartbeat_handler: HeartbeatHandler, return_values: Dic
     excluded_usrdns = set(config_get('tracer-kronos', 'excluded_usrdns').split(','))
     vhost = config_get('tracer-kronos', 'broker_virtual_host', raise_exception=False)
 
-    brokers_alias = [b.strip() for b in config_get('tracer-kronos', 'brokers').split(',')]
+    brokers_alias = config_get_list('tracer-kronos', 'brokers')
     port = config_get_int('tracer-kronos', 'port')
     reconnect_attempts = config_get_int('tracer-kronos', 'reconnect_attempts')
     ssl_key_file = config_get('tracer-kronos', 'ssl_key_file', raise_exception=False)
     ssl_cert_file = config_get('tracer-kronos', 'ssl_cert_file', raise_exception=False)
-    conns = return_values['conns'] = get_stomp_brokers(
+
+    created_conns, _ = stomp_conn_mngr.re_configure(
         brokers=brokers_alias,
         port=port,
         use_ssl=use_ssl,
@@ -367,10 +361,11 @@ def run_once_kronos_file(heartbeat_handler: HeartbeatHandler, return_values: Dic
         ssl_key_file=ssl_key_file,
         ssl_cert_file=ssl_cert_file,
         timeout=sleep_time,
-        logger=logger
+        heartbeats=(0, 5000),
+        logger=logger,
     )
 
-    for conn in conns:
+    for conn in created_conns:
         if not conn.is_connected():
             logger(logging.INFO, 'connecting to %s' % str(conn.transport._Transport__host_and_ports[0]))
             METRICS.counter('reconnect.{host}').labels(host=conn.transport._Transport__host_and_ports[0][0]).inc()


Test patch: