Repository: rucio/rucio

Issue:
Token cleanup issue
Motivation
----------
The query cleaning up tokens is not chunked, leading to an issue for users with lots of tokens
```
-bash-4.2$ rucio-admin rse info SLACXRD_DATADISK
Database exception.
Details: (cx_Oracle.DatabaseError) ORA-01795: maximum number of expressions in a list is 1000
[SQL: DELETE /*+ INDEX(TOKENS_ACCOUNT_EXPIRED_AT_IDX) */ FROM atlas_rucio.tokens WHERE atlas_rucio.tokens.token IN (:token_1_1, :token_1_2, :token_1_3, :token_1_4, :token_1_5, :token_1_6, :token_1_7, :token_1_8, :token_1_9, :token_1_10, :token_1_11, :token_1_12, :token_1_13, :token_1_14, :token_1_15, :token_1_16, :token_1_17, :token_1_18, :token_1_19,
```

Modification
------------
Query needs to be chunked. 
https://github.com/rucio/rucio/blob/4ea8289cf2bc78b7c116afd607e637e2983ffb9e/lib/rucio/core/authentication.py#L486


Discussion:

Resolving patch:
diff --git a/lib/rucio/core/authentication.py b/lib/rucio/core/authentication.py
--- a/lib/rucio/core/authentication.py
+++ b/lib/rucio/core/authentication.py
@@ -27,6 +27,7 @@
 # - Benedikt Ziemons <benedikt.ziemons@cern.ch>, 2020-2021
 # - Rahul Chauhan <omrahulchauhan@gmail.com>, 2021
 # - David Población Criado <david.poblacion.criado@cern.ch>, 2021
+# - Joel Dierkes <joel.dierkes@cern.ch>, 2021
 
 import datetime
 import hashlib
@@ -41,7 +42,7 @@
 from sqlalchemy import and_, or_, select, delete
 
 from rucio.common.exception import CannotAuthenticate, RucioException
-from rucio.common.utils import generate_uuid
+from rucio.common.utils import chunks, generate_uuid
 from rucio.core.account import account_exists
 from rucio.core.oidc import validate_jwt
 from rucio.db.sqla import filter_thread_work
@@ -483,7 +484,8 @@ def __delete_expired_tokens_account(account, session=None):
     for t in result_select.columns('token'):
         tokens.append(t.token)
 
-    stmt_delete = delete(models.Token) \
-        .where(models.Token.token.in_(tokens)) \
-        .prefix_with("/*+ INDEX(TOKENS_ACCOUNT_EXPIRED_AT_IDX) */")
-    session.execute(stmt_delete)
+    for t in chunks(tokens, 100):
+        stmt_delete = delete(models.Token) \
+            .where(models.Token.token.in_(t)) \
+            .prefix_with("/*+ INDEX(TOKENS_ACCOUNT_EXPIRED_AT_IDX) */")
+        session.execute(stmt_delete)


Test patch:
diff --git a/lib/rucio/tests/test_authentication.py b/lib/rucio/tests/test_authentication.py
--- a/lib/rucio/tests/test_authentication.py
+++ b/lib/rucio/tests/test_authentication.py
@@ -26,12 +26,15 @@
 # - Benedikt Ziemons <benedikt.ziemons@cern.ch>, 2020-2021
 # - Rahul Chauhan <omrahulchauhan@gmail.com>, 2021
 # - Simon Fayer <simon.fayer05@imperial.ac.uk>, 2021
+# - Joel Dierkes <joel.dierkes@cern.ch>, 2021
 
 import base64
 import unittest
 
+import datetime
 import pytest
 from requests import session
+import time
 
 from rucio.api.authentication import get_auth_token_user_pass, get_auth_token_ssh, get_ssh_challenge_token, \
     get_auth_token_saml
@@ -40,6 +43,7 @@
 from rucio.common.types import InternalAccount
 from rucio.common.utils import ssh_sign
 from rucio.core.identity import add_account_identity, del_account_identity
+from rucio.db.sqla import models
 from rucio.db.sqla.constants import IdentityType
 from rucio.tests.common import headers, hdrdict, loginhdr, vohdr
 from rucio.tests.common_server import get_vo
@@ -281,3 +285,15 @@ def test_saml_fail(vo, rest_client):
         response = rest_client.get('/auth/saml', headers=headers(hdrdict(headers_dict)))
 
     assert response.status_code == 401
+
+
+@pytest.mark.noparallel(reason='adds many tokens')
+def test_many_tokens(vo, root_account, db_session):
+    """AUTHENTIFICATION (REST): Error when deleting too many tokens."""
+    for i in range(2000):
+        models.Token(account=root_account, token="dummytoken" + str(i), ip='127.0.0.1', expired_at=datetime.datetime.utcnow()).save(session=db_session)
+    db_session.commit()
+
+    # Ensures that the tokens are expired
+    time.sleep(1)
+    print(get_auth_token_user_pass(account='root', username='ddmlab', password='secret', appid='test', ip='127.0.0.1', vo=vo))