Repository: saleor/saleor

Issue:
Migration checking for unique product slugs is slow
When running [0114_auto_20200129_0815.py](https://github.com/mirumee/saleor/blob/a66d99b71fa21381481e6e1d5bfe459580416a1e/saleor/product/migrations/0114_auto_20200129_0815.py) against 20k+ products, this migration will run almost forever.

Instead of what it is doing at the moment, it could simply do this:
```sql
select count(slug), slug from product_product group by slug having count(slug) > 1;
-- Time: 0.021s
```

If duplicates are found, try to enumerate the number of occurrences:
```
A: Slug -> Slug (don't change it)
B: Slug -> Slug-1
C: Slug -> Slug-2
D: Slug -> Slug-3 -> Conflict -> Slug-4


Discussion:

Resolving patch:
diff --git a/saleor/product/migrations/0112_auto_20200129_0050.py b/saleor/product/migrations/0112_auto_20200129_0050.py
--- a/saleor/product/migrations/0112_auto_20200129_0050.py
+++ b/saleor/product/migrations/0112_auto_20200129_0050.py
@@ -1,49 +1,79 @@
 # Generated by Django 2.2.9 on 2020-01-29 06:50
 
+from collections import defaultdict
+
 from django.db import migrations, models
+from django.db.models.functions import Lower
 from django.utils.text import slugify
 
 
 def create_unique_slugs_for_producttypes(apps, schema_editor):
     ProductType = apps.get_model("product", "ProductType")
 
-    for product_type in ProductType.objects.all():
-        if not product_type.slug:
-            product_type.slug = generate_unique_slug(product_type)
-            product_type.save(update_fields=["slug"])
+    product_types = (
+        ProductType.objects.filter(slug__isnull=True).order_by(Lower("name")).iterator()
+    )
+    previous_char = ""
+    slug_values = []
+    for product_type in product_types:
+        first_char = product_type.name[0].lower()
+        if first_char != previous_char:
+            previous_char = first_char
+            slug_values = ProductType.objects.filter(
+                slug__istartswith=first_char
+            ).values_list("slug", flat=True)
+
+        slug = generate_unique_slug(product_type, slug_values)
+        product_type.slug = slug
+        slug_values.append(slug)
+
+
+def generate_unique_slug(instance, slug_values_list):
+    slug = slugify(instance.name)
+    unique_slug = slug
+
+    extension = 1
+
+    while unique_slug in slug_values_list:
+        extension += 1
+        unique_slug = f"{slug}-{extension}"
+
+    return unique_slug
 
 
 def update_non_unique_slugs_for_models(apps, schema_editor):
-    models = ["Category", "Collection"]
+    models_to_update = ["Category", "Collection"]
 
-    for model in models:
+    for model in models_to_update:
         Model = apps.get_model("product", model)
 
-        for instance in Model.objects.all():
-            instance.slug = generate_unique_slug(instance, instance.slug)
-            instance.save(update_fields=["slug"])
+        duplicated_slugs = (
+            Model.objects.all()
+            .values("slug")
+            .annotate(duplicated_slug_num=models.Count("slug"))
+            .filter(duplicated_slug_num__gt=1)
+        )
 
+        slugs_counter = defaultdict(int)
+        for data in duplicated_slugs:
+            slugs_counter[data["slug"]] = data["duplicated_slug_num"]
 
-def generate_unique_slug(instance, slug_value=None):
-    if slug_value:
-        slug = slug_value
-    else:
-        slug = slugify(instance.name)
-    unique_slug = slug
+        queryset = Model.objects.filter(slug__in=slugs_counter.keys()).order_by("name")
 
-    ModelClass = instance.__class__
-    extension = 1
+        for instance in queryset:
+            slugs_counter[instance.slug] -= 1
+            slug = update_slug_to_unique_value(instance.slug, slugs_counter)
+            instance.slug = slug
+            slugs_counter[slug] += 1
 
-    pattern = rf"{slug}-\d+$|{slug}$"
-    slug_values = (
-        ModelClass._default_manager.filter(slug__iregex=pattern)
-        .exclude(pk=instance.pk)
-        .values_list("slug", flat=True)
-    )
 
-    while unique_slug in slug_values:
+def update_slug_to_unique_value(slug_value, slugs_counter):
+    unique_slug = slug_value
+    extension = 1
+
+    while unique_slug in slugs_counter and slugs_counter[unique_slug] > 0:
         extension += 1
-        unique_slug = f"{slug}-{extension}"
+        unique_slug = f"{slug_value}-{extension}"
 
     return unique_slug
 
diff --git a/saleor/product/migrations/0114_auto_20200129_0815.py b/saleor/product/migrations/0114_auto_20200129_0815.py
--- a/saleor/product/migrations/0114_auto_20200129_0815.py
+++ b/saleor/product/migrations/0114_auto_20200129_0815.py
@@ -1,31 +1,36 @@
 # Generated by Django 2.2.9 on 2020-01-29 14:15
 
 from django.db import migrations, models
+from django.db.models.functions import Lower
 from django.utils.text import slugify
 
 
 def create_unique_slug_for_products(apps, schema_editor):
     Product = apps.get_model("product", "Product")
-    for product in Product.objects.all():
-        if not product.slug:
-            product.slug = generate_unique_slug(product)
-            product.save(update_fields=["slug"])
+
+    products = (
+        Product.objects.filter(slug__isnull=True).order_by(Lower("name")).iterator()
+    )
+    previous_char = ""
+    slug_values = []
+    for product in products:
+        first_char = product.name[0].lower()
+        if first_char != previous_char:
+            previous_char = first_char
+            slug_values = Product.objects.filter(
+                slug__istartswith=first_char
+            ).values_list("slug", flat=True)
+
+        slug = generate_unique_slug(product, slug_values)
+        product.slug = slug
+        slug_values.append(slug)
 
 
-def generate_unique_slug(instance):
+def generate_unique_slug(instance, slug_values):
     slug = slugify(instance.name)
     unique_slug = slug
-
-    ModelClass = instance.__class__
     extension = 1
 
-    pattern = rf"{slug}-\d+$|{slug}$"
-    slug_values = (
-        ModelClass._default_manager.filter(slug__iregex=pattern)
-        .exclude(pk=instance.pk)
-        .values_list("slug", flat=True)
-    )
-
     while unique_slug in slug_values:
         extension += 1
         unique_slug = f"{slug}-{extension}"
diff --git a/saleor/warehouse/migrations/0003_warehouse_slug.py b/saleor/warehouse/migrations/0003_warehouse_slug.py
--- a/saleor/warehouse/migrations/0003_warehouse_slug.py
+++ b/saleor/warehouse/migrations/0003_warehouse_slug.py
@@ -1,32 +1,36 @@
 # Generated by Django 2.2.9 on 2020-01-29 06:52
 
 from django.db import migrations, models
+from django.db.models.functions import Lower
 from django.utils.text import slugify
 
 
 def create_unique_slug_for_warehouses(apps, schema_editor):
     Warehouse = apps.get_model("warehouse", "Warehouse")
 
-    for warehouse in Warehouse.objects.all():
-        if not warehouse.slug:
-            warehouse.slug = generate_unique_slug(warehouse)
-            warehouse.save(update_fields=["slug"])
-
-
-def generate_unique_slug(instance):
+    warehouses = (
+        Warehouse.objects.filter(slug__isnull=True).order_by(Lower("name")).iterator()
+    )
+    previous_char = ""
+    slug_values = []
+    for warehouse in warehouses:
+        first_char = warehouse.name[0].lower()
+        if first_char != previous_char:
+            previous_char = first_char
+            slug_values = Warehouse.objects.filter(
+                slug__istartswith=first_char
+            ).values_list("slug", flat=True)
+
+        slug = generate_unique_slug(warehouse, slug_values)
+        warehouse.slug = slug
+        slug_values.append(slug)
+
+
+def generate_unique_slug(instance, slug_values):
     slug = slugify(instance.name)
     unique_slug = slug
-
-    ModelClass = instance.__class__
     extension = 1
 
-    pattern = rf"{slug}-\d+$|{slug}$"
-    slug_values = (
-        ModelClass._default_manager.filter(slug__iregex=pattern)
-        .exclude(pk=instance.pk)
-        .values_list("slug", flat=True)
-    )
-
     while unique_slug in slug_values:
         extension += 1
         unique_slug = f"{slug}-{extension}"


Test patch: