Repository: rucio/rucio

Issue:
Heartbeat cleanup
Motivation
----------
Due to the more dynamic `hostnames` with kubernetes the current cleanup functions of heartbeats are not sufficient, and the table is polluted with old heartbeats which never get removed. Heartbeats do continue to work normally, but the table will grow indefinitely.

Modification
------------
Add a cleanup of all heartbeats older than 24h to the `sanity_check` method.



Discussion:

Resolving patch:
diff --git a/lib/rucio/core/heartbeat.py b/lib/rucio/core/heartbeat.py
--- a/lib/rucio/core/heartbeat.py
+++ b/lib/rucio/core/heartbeat.py
@@ -1,17 +1,25 @@
-# Copyright European Organization for Nuclear Research (CERN)
+# -*- coding: utf-8 -*-
+# Copyright 2015-2021 CERN
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
-# You may not use this file except in compliance with the License.
+# you may not use this file except in compliance with the License.
 # You may obtain a copy of the License at
-# http://www.apache.org/licenses/LICENSE-2.0
 #
-# Authors:
-# - Mario Lassnig, <mario.lassnig@cern.ch>, 2015
-# - Vincent Garonne, <vincent.garonne@cern.ch>, 2015-2017
-# - Hannes Hansen, <hannes.jakob.hansen@cern.ch>, 2018-2019
-# - Cedric Serfon, <cedric.serfon@cern.ch>, 2019
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
 #
-# PY3K COMPATIBLE
+# Authors:
+# - Mario Lassnig <mario.lassnig@cern.ch>, 2015
+# - Vincent Garonne <vincent.garonne@cern.ch>, 2015-2017
+# - Martin Barisits <martin.barisits@cern.ch>, 2016-2019
+# - Hannes Hansen <hannes.jakob.hansen@cern.ch>, 2018-2019
+# - Cedric Serfon <cedric.serfon@cern.ch>, 2019
+# - Radu Carpa <radu.carpa@cern.ch>, 2021
 
 import datetime
 import hashlib
@@ -25,9 +33,12 @@
 from rucio.common.utils import pid_exists
 
 
+DEFAULT_EXPIRATION_DELAY = datetime.timedelta(days=1).total_seconds()
+
+
 @transactional_session
 def sanity_check(executable, hostname, hash_executable=None, pid=None, thread=None,
-                 session=None):
+                 expiration_delay=DEFAULT_EXPIRATION_DELAY, session=None):
     """
     sanity_check wrapper to ignore DatabaseException errors.
 
@@ -36,11 +47,12 @@ def sanity_check(executable, hostname, hash_executable=None, pid=None, thread=No
     :param hash_executable: Hash of the executable.
     :param pid: UNIX Process ID as a number, e.g., 1234.
     :param thread: Python Thread Object.
+    :param expiration_delay: time (in seconds) after which any inactive health check will be removed
     :param session: The database session in use.
     """
     try:
-        _sanity_check(executable=executable, hostname=hostname,
-                      hash_executable=hash_executable, session=session)
+        _sanity_check(executable=executable, hostname=hostname, hash_executable=hash_executable,
+                      expiration_delay=expiration_delay, session=session)
         if pid:
             live(executable=executable, hostname=hostname,
                  pid=pid, thread=thread, session=session)
@@ -49,13 +61,14 @@ def sanity_check(executable, hostname, hash_executable=None, pid=None, thread=No
 
 
 @transactional_session
-def _sanity_check(executable, hostname, hash_executable=None, session=None):
+def _sanity_check(executable, hostname, hash_executable=None, expiration_delay=DEFAULT_EXPIRATION_DELAY, session=None):
     """
     Check if processes on the host are still running.
 
     :param executable: Executable name as a string, e.g., conveyor-submitter.
     :param hostname: Hostname as a string, e.g., rucio-daemon-prod-01.cern.ch.
     :param hash_executable: Hash of the executable.
+    :param expiration_delay: time (in seconds) after which any inactive health check will be removed
     :param session: The database session in use.
     """
     if executable:
@@ -70,6 +83,9 @@ def _sanity_check(executable, hostname, hash_executable=None, session=None):
             if not pid_exists(pid):
                 session.query(Heartbeats).filter_by(hostname=hostname, pid=pid).delete()
 
+    if expiration_delay:
+        cardiac_arrest(older_than=expiration_delay, session=session)
+
 
 @transactional_session
 def live(executable, hostname, pid, thread, older_than=600, hash_executable=None, payload=None, session=None):


Test patch:
diff --git a/lib/rucio/tests/test_heartbeat.py b/lib/rucio/tests/test_heartbeat.py
--- a/lib/rucio/tests/test_heartbeat.py
+++ b/lib/rucio/tests/test_heartbeat.py
@@ -19,14 +19,18 @@
 # - Joaquín Bogado <jbogado@linti.unlp.edu.ar>, 2018
 # - Martin Barisits <martin.barisits@cern.ch>, 2019
 # - Benedikt Ziemons <benedikt.ziemons@cern.ch>, 2020-2021
+# - Radu Carpa <radu.carpa@cern.ch>, 2021
 
 import random
 import threading
 import unittest
+from datetime import datetime, timedelta
 
 import pytest
 
-from rucio.core.heartbeat import live, die, cardiac_arrest, list_payload_counts
+from rucio.core.heartbeat import live, die, cardiac_arrest, list_payload_counts, list_heartbeats, sanity_check
+from rucio.db.sqla.session import transactional_session
+from rucio.db.sqla.models import Heartbeats
 
 
 @pytest.mark.dirty
@@ -38,10 +42,12 @@ def __pid(self):
 
     def __thread(self):
         thread = threading.Thread()
+        self.created_threads.append(thread)
         thread.start()
         return thread
 
     def setUp(self):
+        self.created_threads = []
         cardiac_arrest()
 
     def test_heartbeat_0(self):
@@ -109,5 +115,37 @@ def test_heartbeat_payload(self):
 
         assert list_payload_counts('test5') == {}
 
+    def test_old_heartbeat_cleanup(self):
+        pids = [self.__pid() for _ in range(2)]
+        thread = self.__thread()
+
+        live('test1', 'host0', pids[0], thread)
+        live('test2', 'host0', pids[1], thread)
+        live('test1', 'host1', pids[0], thread)
+        live('test2', 'host1', pids[1], thread)
+        live('test1', 'host2', pids[0], thread)
+        live('test2', 'host2', pids[1], thread)
+
+        assert len(list_heartbeats()) == 6
+
+        @transactional_session
+        def __forge_updated_at(session=None):
+            two_days_ago = datetime.utcnow() - timedelta(days=2)
+            a_dozen_hours_ago = datetime.utcnow() - timedelta(hours=12)
+            session.query(Heartbeats).filter_by(hostname='host1').update({'updated_at': two_days_ago})
+            session.query(Heartbeats).filter_by(hostname='host2').update({'updated_at': a_dozen_hours_ago})
+
+        __forge_updated_at()
+
+        # Default expiration delay. Host1 health checks should get removed.
+        sanity_check(executable=None, hostname=None)
+        assert len(list_heartbeats()) == 4
+
+        # Custom expiration delay. Host2 health checks should get removed too.
+        sanity_check('test2', 'host2', expiration_delay=timedelta(hours=5).total_seconds())
+        assert len(list_heartbeats()) == 2
+
     def tearDown(self):
         cardiac_arrest()
+        for thread in self.created_threads:
+            thread.join()