Repository: jazzband/django-simple-history

Issue:
Second transaction in bulk_create_with_history results in n+1 query
**Describe the bug**
Running `bulk_create_with_history` on a database that doesn't support returning ids from `bulk_create` results in an n+1 query due to all the additional one-off `.filter()` calls it's making

https://github.com/jazzband/django-simple-history/blob/3.1.0/simple_history/utils.py#L113-L131

I believe this was originally needed to maintain the order from the original list, but it should be possible to use the same `CASE/WHEN` trick that Django uses for `bulk_update` to bulk resolve ids in the same order as our original list

https://github.com/django/django/blob/4.0.4/django/db/models/query.py#L624-L638

https://stackoverflow.com/a/19196522/1960509

**To Reproduce**
Steps to reproduce the behavior:
1. Create a model
2. Run a script with `--print-sql` on and bulk create with more than 1 model on a database that lacks support for returning multiple ids from `bulk_create`
3. See more queries than expected

**Expected behavior**

1 query for bulk creation (INSERT INTO normal table) + 1 query for retrieving ids (SELECT WHERE) + 1 query for bulk history creation (INSERT INTO)

**Actual behavior**

1 query for bulk creation (INSERT INTO normal table) + **n** query for retrieving ids (SELECT WHERE * n) + 1 query for bulk history creation (INSERT INTO history table)

**Environment (please complete the following information):**

Not willing to share at this time, I'll file iterate within `django-simple-history` test environment instead



Discussion:

Resolving patch:
diff --git a/simple_history/utils.py b/simple_history/utils.py
--- a/simple_history/utils.py
+++ b/simple_history/utils.py
@@ -2,7 +2,7 @@
 
 import django
 from django.db import transaction
-from django.db.models import ForeignKey, ManyToManyField
+from django.db.models import Case, ForeignKey, ManyToManyField, Q, When
 from django.forms.models import model_to_dict
 
 from simple_history.exceptions import AlternativeManagerError, NotHistoricalModelError
@@ -111,16 +111,35 @@ def bulk_create_with_history(
                 default_date=default_date,
             )
     if second_transaction_required:
-        obj_list = []
         with transaction.atomic(savepoint=False):
-            for obj in objs_with_id:
+            # Generate a common query to avoid n+1 selections
+            #   https://github.com/jazzband/django-simple-history/issues/974
+            cumulative_filter = None
+            obj_when_list = []
+            for i, obj in enumerate(objs_with_id):
                 attributes = dict(
                     filter(
                         lambda x: x[1] is not None,
                         model_to_dict(obj, exclude=exclude_fields).items(),
                     )
                 )
-                obj_list += model_manager.filter(**attributes)
+                q = Q(**attributes)
+                cumulative_filter = (cumulative_filter | q) if cumulative_filter else q
+                # https://stackoverflow.com/a/49625179/1960509
+                # DEV: If an attribute has `then` as a key
+                #   then they'll also run into issues with `bulk_update`
+                #   due to shared implementation
+                #   https://github.com/django/django/blob/4.0.4/django/db/models/query.py#L624-L638
+                obj_when_list.append(When(**attributes, then=i))
+            obj_list = (
+                list(
+                    model_manager.filter(cumulative_filter).order_by(
+                        Case(*obj_when_list)
+                    )
+                )
+                if objs_with_id
+                else []
+            )
             history_manager.bulk_history_create(
                 obj_list,
                 batch_size=batch_size,


Test patch:
diff --git a/simple_history/tests/tests/test_utils.py b/simple_history/tests/tests/test_utils.py
--- a/simple_history/tests/tests/test_utils.py
+++ b/simple_history/tests/tests/test_utils.py
@@ -186,6 +186,38 @@ def test_bulk_create_history_with_duplicates_ignore_conflicts(self):
         self.assertEqual(PollWithUniqueQuestion.objects.count(), 2)
         self.assertEqual(PollWithUniqueQuestion.history.count(), 2)
 
+    def test_bulk_create_history_with_no_ids_return(self):
+        pub_date = timezone.now()
+        objects = [
+            Poll(question="Question 1", pub_date=pub_date),
+            Poll(question="Question 2", pub_date=pub_date),
+            Poll(question="Question 3", pub_date=pub_date),
+            Poll(question="Question 4", pub_date=pub_date),
+            Poll(question="Question 5", pub_date=pub_date),
+        ]
+
+        _bulk_create = Poll._default_manager.bulk_create
+
+        def mock_bulk_create(*args, **kwargs):
+            _bulk_create(*args, **kwargs)
+            return [
+                Poll(question="Question 1", pub_date=pub_date),
+                Poll(question="Question 2", pub_date=pub_date),
+                Poll(question="Question 3", pub_date=pub_date),
+                Poll(question="Question 4", pub_date=pub_date),
+                Poll(question="Question 5", pub_date=pub_date),
+            ]
+
+        with patch.object(
+            Poll._default_manager, "bulk_create", side_effect=mock_bulk_create
+        ):
+            with self.assertNumQueries(3):
+                result = bulk_create_with_history(objects, Poll)
+            self.assertEqual(
+                [poll.question for poll in result], [poll.question for poll in objects]
+            )
+            self.assertNotEqual(result[0].id, None)
+
 
 class BulkCreateWithHistoryTransactionTestCase(TransactionTestCase):
     def setUp(self):
@@ -242,7 +274,7 @@ def test_bulk_create_no_ids_return(self, hist_manager_mock):
         model = Mock(
             _default_manager=Mock(
                 bulk_create=Mock(return_value=[Place(name="Place 1")]),
-                filter=Mock(return_value=objects),
+                filter=Mock(return_value=Mock(order_by=Mock(return_value=objects))),
             ),
             _meta=Mock(get_fields=Mock(return_value=[])),
         )