Repository: django/django
Issue category: Performance Issue

Optimize post-migrate permission creation
Description
		
(last modified by Adam Johnson)		
I have often seen django.contrib.auth.management.create_permissions() take a significant amount of time in test run profiles. It can be optimized by batching more of its operations, including making ContentTypeManager.get_for_models() use batch creation.
For a comparison, I profiled 1518 of Django’s tests in modules called “models”:
$ python -m cProfile -o profile runtests.py --parallel 1 *model*
$ python -m pstats profile <<< 'sort cumtime
stats 10000' | less
Before optimization stats:
Total 11,938,857 function calls taking 5.349 seconds.
88 calls to create_permissions() take 456ms, ~8.5% of the total time.
After optimization stats:
Total 11,359,071 function calls taking 5.035 seconds.
88 calls to create_permissions() now take 239ms, ~4.7% of the toal time.
217ms and 579,786 function calls saved.
Optimization is limited because the post_migrate signal runs once per migrated app config, so there’s no chance to bulk create *all* content types and permissions at once. If we introduced a new “all migrated apps” signal, that could reduce runtime further by batching all creation.


Functions to edit:
django/contrib/auth/management/__init__.py:create_permissions

Patch:
diff --git a/django/contrib/auth/management/__init__.py b/django/contrib/auth/management/__init__.py
index b29a980cb2d5..c40f2aa69dd2 100644
--- a/django/contrib/auth/management/__init__.py
+++ b/django/contrib/auth/management/__init__.py
@@ -46,6 +46,13 @@ def create_permissions(
     if not app_config.models_module:
         return
 
+    try:
+        Permission = apps.get_model("auth", "Permission")
+    except LookupError:
+        return
+    if not router.allow_migrate_model(using, Permission):
+        return
+
     # Ensure that contenttypes are created for this app. Needed if
     # 'django.contrib.auth' is in INSTALLED_APPS before
     # 'django.contrib.contenttypes'.
@@ -62,28 +69,15 @@ def create_permissions(
     try:
         app_config = apps.get_app_config(app_label)
         ContentType = apps.get_model("contenttypes", "ContentType")
-        Permission = apps.get_model("auth", "Permission")
     except LookupError:
         return
 
-    if not router.allow_migrate_model(using, Permission):
-        return
-
-    # This will hold the permissions we're looking for as
-    # (content_type, (codename, name))
-    searched_perms = []
-    # The codenames and ctypes that should exist.
-    ctypes = set()
-    for klass in app_config.get_models():
-        # Force looking up the content types in the current database
-        # before creating foreign keys to them.
-        ctype = ContentType.objects.db_manager(using).get_for_model(
-            klass, for_concrete_model=False
-        )
+    models = list(app_config.get_models())
 
-        ctypes.add(ctype)
-        for perm in _get_all_permissions(klass._meta):
-            searched_perms.append((ctype, perm))
+    # Grab all the ContentTypes.
+    ctypes = ContentType.objects.db_manager(using).get_for_models(
+        *models, for_concrete_models=False
+    )
 
     # Find all the Permissions that have a content_type for a model we're
     # looking for.  We don't need to check for codenames since we already have
@@ -91,20 +85,22 @@ def create_permissions(
     all_perms = set(
         Permission.objects.using(using)
         .filter(
-            content_type__in=ctypes,
+            content_type__in=set(ctypes.values()),
         )
         .values_list("content_type", "codename")
     )
 
     perms = []
-    for ct, (codename, name) in searched_perms:
-        if (ct.pk, codename) not in all_perms:
-            permission = Permission()
-            permission._state.db = using
-            permission.codename = codename
-            permission.name = name
-            permission.content_type = ct
-            perms.append(permission)
+    for model in models:
+        ctype = ctypes[model]
+        for codename, name in _get_all_permissions(model._meta):
+            if (ctype.pk, codename) not in all_perms:
+                permission = Permission()
+                permission._state.db = using
+                permission.codename = codename
+                permission.name = name
+                permission.content_type = ctype
+                perms.append(permission)
 
     Permission.objects.using(using).bulk_create(perms)
     if verbosity >= 2:

Test patch: diff --git a/tests/auth_tests/test_management.py b/tests/auth_tests/test_management.py
index 0cc56b6760d7..5765c500346a 100644
--- a/tests/auth_tests/test_management.py
+++ b/tests/auth_tests/test_management.py
@@ -1528,7 +1528,7 @@ class CreatePermissionsMultipleDatabasesTests(TestCase):
 
     def test_set_permissions_fk_to_using_parameter(self):
         Permission.objects.using("other").delete()
-        with self.assertNumQueries(6, using="other") as captured_queries:
+        with self.assertNumQueries(4, using="other") as captured_queries:
             create_permissions(apps.get_app_config("auth"), verbosity=0, using="other")
         self.assertIn("INSERT INTO", captured_queries[-1]["sql"].upper())
         self.assertGreater(Permission.objects.using("other").count(), 0)