Repository: rucio/rucio

Issue:
Reaper crashes when it tries to delete a DIDs that is a constituent of an archive
Motivation
----------
Reaper crashes when it tries to delete a DIDs that is a constituent of an archive








Discussion:

Resolving patch:
diff --git a/lib/rucio/daemons/atropos/atropos.py b/lib/rucio/daemons/atropos/atropos.py
--- a/lib/rucio/daemons/atropos/atropos.py
+++ b/lib/rucio/daemons/atropos/atropos.py
@@ -23,6 +23,7 @@
 import datetime
 import logging
 import os
+import random
 import socket
 import threading
 import time
@@ -50,7 +51,7 @@
 
 
 def atropos(thread, bulk, date_check, dry_run=True, grace_period=86400,
-            once=True, unlock=False):
+            once=True, unlock=False, spread_period=0):
     """
     Creates an Atropos Worker that gets a list of rules which have an eol_at expired and delete them.
 
@@ -75,6 +76,7 @@ def atropos(thread, bulk, date_check, dry_run=True, grace_period=86400,
     logging.debug(prepend_str + 'Starting worker')
     summary = {}
     lifetime_exceptions = {}
+    rand = random.Random(hb['assign_thread'])
     for excep in rucio.core.lifetime_exception.list_exceptions(exception_id=None, states=[LifetimeExceptionsState.APPROVED, ], session=None):
         if '%s:%s' % (excep['scope'], excep['name']) not in lifetime_exceptions:
             lifetime_exceptions['%s:%s' % (excep['scope'], excep['name'])] = excep['expires_at']
@@ -145,8 +147,9 @@ def atropos(thread, bulk, date_check, dry_run=True, grace_period=86400,
                         if no_locks:
                             logging.warning(prepend_str + 'Cannot find a lock for rule %s on DID %s' % (rule.id, did))
                         if not dry_run:
-                            logging.info(prepend_str + 'Setting %s seconds lifetime for rule %s' % (grace_period, rule.id))
-                            options = {'lifetime': grace_period}
+                            lifetime = grace_period + rand.randrange(spread_period + 1)
+                            logging.info(prepend_str + 'Setting %s seconds lifetime for rule %s' % (lifetime, rule.id))
+                            options = {'lifetime': lifetime}
                             if rule.locked and unlock:
                                 logging.info(prepend_str + 'Unlocking rule %s', rule.id)
                                 options['locked'] = False
@@ -182,7 +185,7 @@ def atropos(thread, bulk, date_check, dry_run=True, grace_period=86400,
 
 
 def run(threads=1, bulk=100, date_check=None, dry_run=True, grace_period=86400,
-        once=True, unlock=False):
+        once=True, unlock=False, spread_period=0):
     """
     Starts up the atropos threads.
     """
@@ -199,7 +202,8 @@ def run(threads=1, bulk=100, date_check=None, dry_run=True, grace_period=86400,
                                                             'dry_run': dry_run,
                                                             'grace_period': grace_period,
                                                             'bulk': bulk,
-                                                            'unlock': unlock}) for i in range(0, threads)]
+                                                            'unlock': unlock,
+                                                            'spread_period': spread_period}) for i in range(0, threads)]
     [t.start() for t in thread_list]
 
     logging.info('waiting for interrupts')
diff --git a/lib/rucio/daemons/hermes/hermes.py b/lib/rucio/daemons/hermes/hermes.py
--- a/lib/rucio/daemons/hermes/hermes.py
+++ b/lib/rucio/daemons/hermes/hermes.py
@@ -14,7 +14,7 @@
 #
 # Authors:
 # - Mario Lassnig <mario.lassnig@cern.ch>, 2014-2018
-# - Thomas Beermann <thomas.beermann@cern.ch>, 2014
+# - Thomas Beermann <thomas.beermann@cern.ch>, 2014-2019
 # - Wen Guan <wguan.icedew@gmail.com>, 2014-2015
 # - Vincent Garonne <vgaronne@gmail.com>, 2015-2018
 # - Martin Barisits <martin.barisits@cern.ch>, 2016-2017
@@ -356,6 +356,12 @@ def deliver_messages(once=False, brokers_resolved=None, thread=0, bulk=1000, del
                           heartbeat['assign_thread'], heartbeat['nr_threads'], t_delay)
         time.sleep(t_delay)
 
+    for conn in conns:
+        try:
+            conn.disconnect()
+        except Exception:
+            pass
+
     logging.debug('[broker] %i:%i - graceful stop requested',
                   heartbeat['assign_thread'], heartbeat['nr_threads'])
 
diff --git a/setup_rucio.py b/setup_rucio.py
--- a/setup_rucio.py
+++ b/setup_rucio.py
@@ -52,7 +52,6 @@
 requirements_files = ['tools/pip-requires', 'tools/pip-requires-client']
 data_files = [('rucio/etc/', glob.glob('etc/*.template')),
               ('rucio/etc/web', glob.glob('etc/web/*.template')),
-              ('rucio/etc/schemas', glob.glob('etc/schemas/*.json')),
               ('rucio/tools/', ['tools/pip-requires', 'tools/pip-requires-client', 'tools/pip-requires-test',
                                 'tools/bootstrap.py', 'tools/reset_database.py']),
               ('rucio/etc/mail_templates/', glob.glob('etc/mail_templates/*.tmpl'))]


Test patch: