Repository: comic/grand-challenge.org

Issue:
Move `wait_for_build_completion` to acks late micro short queue and use delays
`wait_for_build_completion` uses a countdown that causes the same issues described in https://github.com/comic/grand-challenge.org/blob/ce130d451034377b0f3dfd83590268e48a66d518/app/grandchallenge/components/tasks.py#L255-L271. 

This task is not resource-intensive so:
- If the task is not idempotent it should be made so
- It should be run on the acks-late-micro-short queue
- Retries should use the same strategy as https://github.com/comic/grand-challenge.org/blob/ce130d451034377b0f3dfd83590268e48a66d518/app/grandchallenge/components/tasks.py#L323-L327


Discussion:

Resolving patch:
diff --git a/app/grandchallenge/codebuild/models.py b/app/grandchallenge/codebuild/models.py
--- a/app/grandchallenge/codebuild/models.py
+++ b/app/grandchallenge/codebuild/models.py
@@ -65,6 +65,7 @@ def refresh_logs(self):
             self.build_log = "Log file not available."
 
     def add_image_to_algorithm(self):
+        # TODO, this would be much faster using S3 copy, can then run on a smaller queue
         with private_s3_storage.open(
             f"codebuild/artifacts/{self.build_number}/{self.build_config['projectName']}/container-image.tar.gz"
         ) as file:
diff --git a/app/grandchallenge/codebuild/tasks.py b/app/grandchallenge/codebuild/tasks.py
--- a/app/grandchallenge/codebuild/tasks.py
+++ b/app/grandchallenge/codebuild/tasks.py
@@ -1,12 +1,15 @@
 from celery import shared_task
 from django.apps import apps
+from django.conf import settings
 from django.core.exceptions import ObjectDoesNotExist
+from django.db import transaction
 from django.db.transaction import on_commit
 
 from grandchallenge.algorithms.models import Algorithm, AlgorithmImage
+from grandchallenge.components.tasks import _retry
 
 
-@shared_task()
+@shared_task(**settings.CELERY_TASK_DECORATOR_KWARGS["acks-late-micro-short"])
 def create_codebuild_build(*, pk):
     GitHubWebhookMessage = apps.get_model(  # noqa: N806
         app_label="github", model_name="GitHubWebhookMessage"
@@ -29,50 +32,59 @@ def create_codebuild_build(*, pk):
         # Repository is not linked to algorithm
         return
 
-    algorithm_image = AlgorithmImage.objects.create(
-        algorithm=algorithm,
-        requires_gpu=algorithm.image_requires_gpu,
-        requires_memory_gb=algorithm.image_requires_memory_gb,
-    )
-
-    build = Build.objects.create(
-        webhook_message=ghwm, algorithm_image=algorithm_image,
-    )
+    with transaction.atomic():
+        algorithm_image = AlgorithmImage.objects.create(
+            algorithm=algorithm,
+            requires_gpu=algorithm.image_requires_gpu,
+            requires_memory_gb=algorithm.image_requires_memory_gb,
+        )
+        build = Build.objects.create(
+            webhook_message=ghwm, algorithm_image=algorithm_image,
+        )
 
-    on_commit(
-        lambda: wait_for_build_completion.apply_async(
-            kwargs={"build_pk": str(build.pk)}
+        on_commit(
+            lambda: wait_for_build_completion.apply_async(
+                kwargs={"build_pk": str(build.pk)}
+            )
         )
-    )
 
 
-@shared_task(bind=True, max_retries=120)
-def wait_for_build_completion(self, *, build_pk):
+@shared_task(**settings.CELERY_TASK_DECORATOR_KWARGS["acks-late-micro-short"])
+def wait_for_build_completion(*, build_pk, retries=0):
     Build = apps.get_model(  # noqa: N806
         app_label="codebuild", model_name="Build"
     )
 
     build = Build.objects.get(pk=build_pk)
-    build.refresh_status()
-
-    if build.status == build.BuildStatusChoices.IN_PROGRESS:
-        self.retry(countdown=30)
-    else:
-        build.refresh_logs()
-        build.save()
-        if build.status == build.BuildStatusChoices.SUCCEEDED:
-            on_commit(
-                lambda: add_image_to_algorithm.apply_async(
-                    kwargs={"build_pk": str(build_pk)}
-                )
+
+    with transaction.atomic():
+        build.refresh_status()
+
+        if build.status == build.BuildStatusChoices.IN_PROGRESS:
+            _retry(
+                task=wait_for_build_completion,
+                signature_kwargs={"kwargs": {"build_pk": build_pk}},
+                retries=retries,
             )
+            return
+        else:
+            build.refresh_logs()
+            build.save()
+            if build.status == build.BuildStatusChoices.SUCCEEDED:
+                on_commit(
+                    lambda: add_image_to_algorithm.apply_async(
+                        kwargs={"build_pk": str(build_pk)}
+                    )
+                )
 
 
-@shared_task
+@shared_task(**settings.CELERY_TASK_DECORATOR_KWARGS["acks-late-2xlarge"])
 def add_image_to_algorithm(*, build_pk):
     Build = apps.get_model(  # noqa: N806
         app_label="codebuild", model_name="Build"
     )
 
     build = Build.objects.get(pk=build_pk)
-    build.add_image_to_algorithm()
+
+    if not build.algorithm_image.image:
+        build.add_image_to_algorithm()


Test patch: