Repository: rucio/rucio

Issue:
Deadlock in token expiration
Motivation
----------
In each token request the authentication core removes all already expired tokens of the account. For accounts (e.g. panda, pilot) with many tokens, this currently leads to deadlocks. It is not clear if there is some query degradation, this is currently being investigated. This is the first time we are encountering this issue.

To circumvent this issue, we currently disabled the token expiration. 
https://github.com/rucio/rucio/blob/342e0831ca4088dbb3b4f256133469b8b85c5d8c/lib/rucio/core/authentication.py#L152
The tokens lifetime is correctly validated, so this only leads to tokens being archived for longer.

We should change this expiration to use `for_update(skip_locked=True)` or do the token cleanup entirely in a different process.


Discussion:

Resolving patch:
diff --git a/lib/rucio/core/authentication.py b/lib/rucio/core/authentication.py
--- a/lib/rucio/core/authentication.py
+++ b/lib/rucio/core/authentication.py
@@ -14,7 +14,7 @@
 #
 # Authors:
 # - Mario Lassnig <mario@lassnig.net>, 2012-2018
-# - Martin Barisits <martin.barisits@cern.ch>, 2012-2017
+# - Martin Barisits <martin.barisits@cern.ch>, 2012-2019
 # - Vincent Garonne <vgaronne@gmail.com>, 2012-2017
 # - Angelos Molfetas <Angelos.Molfetas@cern.ch>, 2012
 # - Ralph Vigne <ralph.vigne@cern.ch>, 2013-2014
@@ -114,7 +114,7 @@ def get_auth_token_user_pass(account, username, password, appid, ip=None, sessio
 
     # remove expired tokens
     session.query(models.Token).filter(models.Token.expired_at < datetime.datetime.utcnow(),
-                                       models.Token.account == account).delete()
+                                       models.Token.account == account).with_for_update(skip_locked=True).delete()
 
     # create new rucio-auth-token for account
     tuid = generate_uuid()  # NOQA
@@ -150,7 +150,7 @@ def get_auth_token_x509(account, dn, appid, ip=None, session=None):
 
     # remove expired tokens
     session.query(models.Token).filter(models.Token.expired_at < datetime.datetime.utcnow(),
-                                       models.Token.account == account).delete()
+                                       models.Token.account == account).with_for_update(skip_locked=True).delete()
 
     # create new rucio-auth-token for account
     tuid = generate_uuid()  # NOQA
@@ -186,7 +186,7 @@ def get_auth_token_gss(account, gsstoken, appid, ip=None, session=None):
 
     # remove expired tokens
     session.query(models.Token).filter(models.Token.expired_at < datetime.datetime.utcnow(),
-                                       models.Token.account == account).delete()
+                                       models.Token.account == account).with_for_update(skip_locked=True).delete()
 
     # create new rucio-auth-token for account
     tuid = generate_uuid()  # NOQA
@@ -250,7 +250,7 @@ def get_auth_token_ssh(account, signature, appid, ip=None, session=None):
 
     # remove expired tokens
     session.query(models.Token).filter(models.Token.expired_at < datetime.datetime.utcnow(),
-                                       models.Token.account == account).delete()
+                                       models.Token.account == account).with_for_update(skip_locked=True).delete()
 
     # create new rucio-auth-token for account
     tuid = generate_uuid()  # NOQA
diff --git a/lib/rucio/daemons/atropos/atropos.py b/lib/rucio/daemons/atropos/atropos.py
--- a/lib/rucio/daemons/atropos/atropos.py
+++ b/lib/rucio/daemons/atropos/atropos.py
@@ -51,7 +51,7 @@
 
 
 def atropos(thread, bulk, date_check, dry_run=True, grace_period=86400,
-            once=True, unlock=False, spread_period=0):
+            once=True, unlock=False, spread_period=0, purge_replicas=False):
     """
     Creates an Atropos Worker that gets a list of rules which have an eol_at expired and delete them.
 
@@ -150,6 +150,8 @@ def atropos(thread, bulk, date_check, dry_run=True, grace_period=86400,
                             lifetime = grace_period + rand.randrange(spread_period + 1)
                             logging.info(prepend_str + 'Setting %s seconds lifetime for rule %s' % (lifetime, rule.id))
                             options = {'lifetime': lifetime}
+                            if purge_replicas:
+                                options = {'purge_replicas': True}
                             if rule.locked and unlock:
                                 logging.info(prepend_str + 'Unlocking rule %s', rule.id)
                                 options['locked'] = False
@@ -185,7 +187,7 @@ def atropos(thread, bulk, date_check, dry_run=True, grace_period=86400,
 
 
 def run(threads=1, bulk=100, date_check=None, dry_run=True, grace_period=86400,
-        once=True, unlock=False, spread_period=0):
+        once=True, unlock=False, spread_period=0, purge_replicas=False):
     """
     Starts up the atropos threads.
     """
@@ -203,7 +205,8 @@ def run(threads=1, bulk=100, date_check=None, dry_run=True, grace_period=86400,
                                                             'grace_period': grace_period,
                                                             'bulk': bulk,
                                                             'unlock': unlock,
-                                                            'spread_period': spread_period}) for i in range(0, threads)]
+                                                            'spread_period': spread_period,
+                                                            'purge_replicas': purge_replicas}) for i in range(0, threads)]
     [t.start() for t in thread_list]
 
     logging.info('waiting for interrupts')


Test patch: