Repository: comic/grand-challenge.org

Issue:
Reduce database calls for `update_challenge_results_cache()`



Discussion:

Resolving patch:
diff --git a/app/grandchallenge/challenges/tasks.py b/app/grandchallenge/challenges/tasks.py
--- a/app/grandchallenge/challenges/tasks.py
+++ b/app/grandchallenge/challenges/tasks.py
@@ -1,5 +1,7 @@
 from celery import shared_task
+from django.contrib.auth import get_user_model
 from django.core.mail import mail_managers
+from django.db.models import Count, Max
 from requests import exceptions, get
 
 from grandchallenge.challenges.models import Challenge, ExternalChallenge
@@ -9,29 +11,47 @@
 
 @shared_task
 def update_challenge_results_cache():
-    for c in Challenge.objects.all():
-        kwargs = {
-            "cached_num_participants": c.participants_group.user_set.all().count()
-        }
-
-        challenge_results = Evaluation.objects.filter(
-            submission__phase__challenge=c, published=True
-        ).order_by("-created")
+    challenges = Challenge.objects.all()
+    evaluation_info = (
+        Evaluation.objects.filter(published=True)
+        .values("submission__phase__challenge_id")
+        .annotate(
+            cached_num_results=Count("submission__phase__challenge_id"),
+            cached_latest_result=Max("created"),
+        )
+    )
+    evaluation_info_by_challenge = {
+        str(v["submission__phase__challenge_id"]): v for v in evaluation_info
+    }
+    participant_counts = (
+        get_user_model()
+        .objects.values("groups__participants_of_challenge")
+        .annotate(cached_num_participants=Count("pk"))
+    )
+    participant_counts_by_challenge = {
+        str(v["groups__participants_of_challenge"]): v
+        for v in participant_counts
+    }
 
-        try:
-            kwargs.update(
-                {
-                    "cached_num_results": challenge_results.count(),
-                    "cached_latest_result": challenge_results.first().created,
-                }
-            )
-        except AttributeError:
-            # No results for this challenge
-            kwargs.update(
-                {"cached_num_results": 0, "cached_latest_result": None}
-            )
+    for c in challenges:
+        c.cached_num_results = evaluation_info_by_challenge.get(
+            str(c.pk), {}
+        ).get("cached_num_results", 0)
+        c.cached_latest_result = evaluation_info_by_challenge.get(
+            str(c.pk), {}
+        ).get("cached_latest_result", None)
+        c.cached_num_participants = participant_counts_by_challenge.get(
+            str(c.pk), {}
+        ).get("cached_num_participants", 0)
 
-        Challenge.objects.filter(pk=c.pk).update(**kwargs)
+    Challenge.objects.bulk_update(
+        challenges,
+        [
+            "cached_num_results",
+            "cached_num_participants",
+            "cached_latest_result",
+        ],
+    )
 
 
 @shared_task


Test patch:
diff --git a/app/tests/challenges_tests/test_tasks.py b/app/tests/challenges_tests/test_tasks.py
new file mode 100644
--- /dev/null
+++ b/app/tests/challenges_tests/test_tasks.py
@@ -0,0 +1,32 @@
+import pytest
+
+from grandchallenge.challenges.tasks import update_challenge_results_cache
+from tests.evaluation_tests.factories import EvaluationFactory
+from tests.factories import ChallengeFactory
+
+
+@pytest.mark.django_db
+def test_challenge_update(
+    client, two_challenge_sets, django_assert_num_queries
+):
+    c1 = two_challenge_sets.challenge_set_1.challenge
+    c2 = two_challenge_sets.challenge_set_2.challenge
+
+    _ = EvaluationFactory(
+        submission__phase__challenge=c1, method__phase__challenge=c1
+    )
+    _ = EvaluationFactory(
+        submission__phase__challenge=c2, method__phase__challenge=c2
+    )
+
+    with django_assert_num_queries(4) as _:
+        update_challenge_results_cache()
+
+    # check the # queries stays the same even with more challenges & evaluations
+
+    c3 = ChallengeFactory()
+    _ = EvaluationFactory(
+        submission__phase__challenge=c3, method__phase__challenge=c3
+    )
+    with django_assert_num_queries(4) as _:
+        update_challenge_results_cache()