Repository: jazzband/django-oauth-toolkit

Issue:
`cleartokens` management command could have some performance improvements
I'm using the `cleartokens` command to clear the expired tokens from my MySQL database and try to improve performance. However, we got to a point where we have a lot of tokens in our database (~300 MM) because of a bug in another application.

The command no longer workers because the `delete()` method from Django tries to load the entire object list to the python memory. The task will make the machine CPU goes to 100% and won't be able to delete anything.

I think we should have some alternatives to clear the tokens. Maybe some kind of pagination so it would delete X tokens at a time or maybe using the `_raw_delete()` method from Django ORM.

Is that an acceptable resolution for the problem? I can try and work in a PR to address this issue.



Discussion:
@jarussi-luizalabs I think it´s a good feature a expired token "pruning" by date. Currently i working on it. I hope see my work PR merged. 

Obg.

Resolving patch:
diff --git a/oauth2_provider/models.py b/oauth2_provider/models.py
--- a/oauth2_provider/models.py
+++ b/oauth2_provider/models.py
@@ -1,4 +1,5 @@
 import logging
+import time
 import uuid
 from datetime import timedelta
 from urllib.parse import parse_qsl, urlparse
@@ -621,12 +622,31 @@ def get_refresh_token_admin_class():
 
 
 def clear_expired():
+    def batch_delete(queryset, query):
+        CLEAR_EXPIRED_TOKENS_BATCH_SIZE = oauth2_settings.CLEAR_EXPIRED_TOKENS_BATCH_SIZE
+        CLEAR_EXPIRED_TOKENS_BATCH_INTERVAL = oauth2_settings.CLEAR_EXPIRED_TOKENS_BATCH_INTERVAL
+        current_no = start_no = queryset.count()
+
+        while current_no:
+            flat_queryset = queryset.values_list("id", flat=True)[:CLEAR_EXPIRED_TOKENS_BATCH_SIZE]
+            batch_length = flat_queryset.count()
+            queryset.model.objects.filter(id__in=list(flat_queryset)).delete()
+            logger.debug(f"{batch_length} tokens deleted, {current_no-batch_length} left")
+            queryset = queryset.model.objects.filter(query)
+            time.sleep(CLEAR_EXPIRED_TOKENS_BATCH_INTERVAL)
+            current_no = queryset.count()
+
+        stop_no = queryset.model.objects.filter(query).count()
+        deleted = start_no - stop_no
+        return deleted
+
     now = timezone.now()
     refresh_expire_at = None
     access_token_model = get_access_token_model()
     refresh_token_model = get_refresh_token_model()
     grant_model = get_grant_model()
     REFRESH_TOKEN_EXPIRE_SECONDS = oauth2_settings.REFRESH_TOKEN_EXPIRE_SECONDS
+
     if REFRESH_TOKEN_EXPIRE_SECONDS:
         if not isinstance(REFRESH_TOKEN_EXPIRE_SECONDS, timedelta):
             try:
@@ -636,31 +656,32 @@ def clear_expired():
                 raise ImproperlyConfigured(e)
         refresh_expire_at = now - REFRESH_TOKEN_EXPIRE_SECONDS
 
-    with transaction.atomic():
-        if refresh_expire_at:
-            revoked = refresh_token_model.objects.filter(
-                revoked__lt=refresh_expire_at,
-            )
-            expired = refresh_token_model.objects.filter(
-                access_token__expires__lt=refresh_expire_at,
-            )
+    if refresh_expire_at:
+        revoked_query = models.Q(revoked__lt=refresh_expire_at)
+        revoked = refresh_token_model.objects.filter(revoked_query)
+
+        revoked_deleted_no = batch_delete(revoked, revoked_query)
+        logger.info("%s Revoked refresh tokens deleted", revoked_deleted_no)
+
+        expired_query = models.Q(access_token__expires__lt=refresh_expire_at)
+        expired = refresh_token_model.objects.filter(expired_query)
 
-            logger.info("%s Revoked refresh tokens to be deleted", revoked.count())
-            logger.info("%s Expired refresh tokens to be deleted", expired.count())
+        expired_deleted_no = batch_delete(expired, expired_query)
+        logger.info("%s Expired refresh tokens deleted", expired_deleted_no)
+    else:
+        logger.info("refresh_expire_at is %s. No refresh tokens deleted.", refresh_expire_at)
 
-            revoked.delete()
-            expired.delete()
-        else:
-            logger.info("refresh_expire_at is %s. No refresh tokens deleted.", refresh_expire_at)
+    access_token_query = models.Q(refresh_token__isnull=True, expires__lt=now)
+    access_tokens = access_token_model.objects.filter(access_token_query)
 
-        access_tokens = access_token_model.objects.filter(refresh_token__isnull=True, expires__lt=now)
-        grants = grant_model.objects.filter(expires__lt=now)
+    access_tokens_delete_no = batch_delete(access_tokens, access_token_query)
+    logger.info("%s Expired access tokens deleted", access_tokens_delete_no)
 
-        logger.info("%s Expired access tokens to be deleted", access_tokens.count())
-        logger.info("%s Expired grant tokens to be deleted", grants.count())
+    grants_query = models.Q(expires__lt=now)
+    grants = grant_model.objects.filter(grants_query)
 
-        access_tokens.delete()
-        grants.delete()
+    grants_deleted_no = batch_delete(grants, grants_query)
+    logger.info("%s Expired grant tokens deleted", grants_deleted_no)
 
 
 def redirect_to_uri_allowed(uri, allowed_uris):
diff --git a/oauth2_provider/settings.py b/oauth2_provider/settings.py
--- a/oauth2_provider/settings.py
+++ b/oauth2_provider/settings.py
@@ -101,6 +101,8 @@
     # Whether to re-create OAuthlibCore on every request.
     # Should only be required in testing.
     "ALWAYS_RELOAD_OAUTHLIB_CORE": False,
+    "CLEAR_EXPIRED_TOKENS_BATCH_SIZE": 10000,
+    "CLEAR_EXPIRED_TOKENS_BATCH_INTERVAL": 0.1,
 }
 
 # List of settings that cannot be empty


Test patch:
diff --git a/tests/settings.py b/tests/settings.py
--- a/tests/settings.py
+++ b/tests/settings.py
@@ -156,3 +156,6 @@
 OAUTH2_PROVIDER_APPLICATION_MODEL = "oauth2_provider.Application"
 OAUTH2_PROVIDER_REFRESH_TOKEN_MODEL = "oauth2_provider.RefreshToken"
 OAUTH2_PROVIDER_ID_TOKEN_MODEL = "oauth2_provider.IDToken"
+
+CLEAR_EXPIRED_TOKENS_BATCH_SIZE = 1
+CLEAR_EXPIRED_TOKENS_BATCH_INTERVAL = 0