Repository: pytest-dev/pytest-django

Issue:
on_commit support
It would be nice if pytest-django had on_commit support that didn't require a django_db(transaction=True) mark, something like this maybe.
https://medium.com/gitux/speed-up-django-transaction-hooks-tests-6de4a558ef96


Discussion:
For my particular usecase my task system submits tasks in on commit. So if we error out of a transaction we don't run any associated tasks.
In tests I have helpers to run some or all of the queued events.
In those helpers I also have an assert that fails when in a transaction, this exists to catch being in a test but no a transaction test, because I have on so many occasions spent so much time trying to work out where the hell my task has disappeared to.

It would be nice if there was a django_db variant that just ran the on_commit code at the same place it would run if transaction=True. It would also be nice if it had roughly equivalent performance as transaction=False so I could just blanket use it everywhere.
FWIW for testing a scenario where `on_commit` calls `mytask.delay()`,  I was able to validate the task invocation by patching `on_commit` like so:

```
with mock.patch('mypackage.tasks.mytask.delay') as mock_task_delay:
    with mock.patch('mypackage.models.transaction.on_commit', new=lambda fn: fn()):
        mymodel.save()
        assert mock_task_delay.call_count == 1
        ...
```

Since I created this issue I've also come across https://pypi.org/project/django-capture-on-commit-callbacks/

@trawick if you just want to validate that the call happens that is a simple and effective soution.
If you want to actually run the task code it has the downside that the task is scheduled immediately and so depending on how tasks are run in your tests it might fail intermittently if it depends on operations done later in the transaction.
Django 3.2 added [`TestCase.captureOnCommitCallbacks`](https://docs.djangoproject.com/en/3.2/topics/testing/tools/#django.test.TestCase.captureOnCommitCallbacks). It should be straightforward enough to add to pytest-django. I would use the `django_assert_num_queries` fixture as a template.

I would be happy for a PR if anyone would like to work on it!

Resolving patch:
diff --git a/pytest_django/fixtures.py b/pytest_django/fixtures.py
--- a/pytest_django/fixtures.py
+++ b/pytest_django/fixtures.py
@@ -1,5 +1,5 @@
 """All pytest-django fixtures"""
-from typing import Any, Generator, Iterable, List, Optional, Tuple, Union
+from typing import Any, Callable, Generator, Iterable, List, Optional, Tuple, Union
 import os
 from contextlib import contextmanager
 from functools import partial
@@ -8,7 +8,7 @@
 
 from . import live_server_helper
 from .django_compat import is_django_unittest
-from .lazy_django import skip_if_no_django
+from .lazy_django import skip_if_no_django, get_django_version
 
 TYPE_CHECKING = False
 if TYPE_CHECKING:
@@ -38,6 +38,7 @@
     "_live_server_helper",
     "django_assert_num_queries",
     "django_assert_max_num_queries",
+    "django_capture_on_commit_callbacks",
 ]
 
 
@@ -542,3 +543,38 @@ def django_assert_num_queries(pytestconfig):
 @pytest.fixture(scope="function")
 def django_assert_max_num_queries(pytestconfig):
     return partial(_assert_num_queries, pytestconfig, exact=False)
+
+
+@contextmanager
+def _capture_on_commit_callbacks(
+    *,
+    using: Optional[str] = None,
+    execute: bool = False
+):
+    from django.db import DEFAULT_DB_ALIAS, connections
+    from django.test import TestCase
+
+    if using is None:
+        using = DEFAULT_DB_ALIAS
+
+    # Polyfill of Django code as of Django 3.2.
+    if get_django_version() < (3, 2):
+        callbacks = []  # type: List[Callable[[], Any]]
+        start_count = len(connections[using].run_on_commit)
+        try:
+            yield callbacks
+        finally:
+            run_on_commit = connections[using].run_on_commit[start_count:]
+            callbacks[:] = [func for sids, func in run_on_commit]
+            if execute:
+                for callback in callbacks:
+                    callback()
+
+    else:
+        with TestCase.captureOnCommitCallbacks(using=using, execute=execute) as callbacks:
+            yield callbacks
+
+
+@pytest.fixture(scope="function")
+def django_capture_on_commit_callbacks():
+    return _capture_on_commit_callbacks
diff --git a/pytest_django/plugin.py b/pytest_django/plugin.py
--- a/pytest_django/plugin.py
+++ b/pytest_django/plugin.py
@@ -25,6 +25,7 @@
 from .fixtures import django_db_modify_db_settings_parallel_suffix  # noqa
 from .fixtures import django_db_modify_db_settings_tox_suffix  # noqa
 from .fixtures import django_db_modify_db_settings_xdist_suffix  # noqa
+from .fixtures import django_capture_on_commit_callbacks  # noqa
 from .fixtures import _live_server_helper  # noqa
 from .fixtures import admin_client  # noqa
 from .fixtures import admin_user  # noqa


Test patch:
diff --git a/tests/test_fixtures.py b/tests/test_fixtures.py
--- a/tests/test_fixtures.py
+++ b/tests/test_fixtures.py
@@ -230,6 +230,70 @@ def test_queries(django_assert_num_queries):
     assert result.ret == 1
 
 
+@pytest.mark.django_db
+def test_django_capture_on_commit_callbacks(django_capture_on_commit_callbacks) -> None:
+    if not connection.features.supports_transactions:
+        pytest.skip("transactions required for this test")
+
+    scratch = []
+    with django_capture_on_commit_callbacks() as callbacks:
+        transaction.on_commit(lambda: scratch.append("one"))
+    assert len(callbacks) == 1
+    assert scratch == []
+    callbacks[0]()
+    assert scratch == ["one"]
+
+    scratch = []
+    with django_capture_on_commit_callbacks(execute=True) as callbacks:
+        transaction.on_commit(lambda: scratch.append("two"))
+        transaction.on_commit(lambda: scratch.append("three"))
+    assert len(callbacks) == 2
+    assert scratch == ["two", "three"]
+    callbacks[0]()
+    assert scratch == ["two", "three", "two"]
+
+
+@pytest.mark.django_db(databases=["default", "second"])
+def test_django_capture_on_commit_callbacks_multidb(django_capture_on_commit_callbacks) -> None:
+    if not connection.features.supports_transactions:
+        pytest.skip("transactions required for this test")
+
+    scratch = []
+    with django_capture_on_commit_callbacks(using="default", execute=True) as callbacks:
+        transaction.on_commit(lambda: scratch.append("one"))
+    assert len(callbacks) == 1
+    assert scratch == ["one"]
+
+    scratch = []
+    with django_capture_on_commit_callbacks(using="second", execute=True) as callbacks:
+        transaction.on_commit(lambda: scratch.append("two"))  # pragma: no cover
+    assert len(callbacks) == 0
+    assert scratch == []
+
+    scratch = []
+    with django_capture_on_commit_callbacks(using="default", execute=True) as callbacks:
+        transaction.on_commit(lambda: scratch.append("ten"))
+        transaction.on_commit(lambda: scratch.append("twenty"), using="second")  # pragma: no cover
+        transaction.on_commit(lambda: scratch.append("thirty"))
+    assert len(callbacks) == 2
+    assert scratch == ["ten", "thirty"]
+
+
+@pytest.mark.django_db(transaction=True)
+def test_django_capture_on_commit_callbacks_transactional(
+    django_capture_on_commit_callbacks,
+) -> None:
+    if not connection.features.supports_transactions:
+        pytest.skip("transactions required for this test")
+
+    # Bad usage: no transaction (executes immediately).
+    scratch = []
+    with django_capture_on_commit_callbacks() as callbacks:
+        transaction.on_commit(lambda: scratch.append("one"))
+    assert len(callbacks) == 0
+    assert scratch == ["one"]
+
+
 class TestSettings:
     """Tests for the settings fixture, order matters"""