Repository: comic/grand-challenge.org

Issue:
Remove countdown in build_images 
This task should work synchronously, and not rely on a countdown timer. Maybe create all the `RawImageFile`s first, then add these at the create time of `RawImageUploadSession`. Or, don't rely on a post_save signal, but manually invoke this when all of the database objects are ready.

This will then simplify the test code.


Discussion:

Resolving patch:
diff --git a/app/grandchallenge/cases/forms.py b/app/grandchallenge/cases/forms.py
--- a/app/grandchallenge/cases/forms.py
+++ b/app/grandchallenge/cases/forms.py
@@ -33,24 +33,29 @@ class UploadRawImagesForm(forms.ModelForm):
     )
 
     def __init__(self, *args, **kwargs):
-        super(UploadRawImagesForm, self).__init__(*args, **kwargs)
+        super().__init__(*args, **kwargs)
         self.helper = FormHelper()
         self.helper.add_input(Submit("save", "Submit"))
 
     def save(self, commit=True):
-        with transaction.atomic():
-            instance = super(UploadRawImagesForm, self).save(commit=commit)
-
-            # Create links between the created session and all uploaded files
-            uploaded_files = self.cleaned_data[
-                "files"]  # type: List[StagedAjaxFile]
-
-            for uploaded_file in uploaded_files:
-                RawImageFile.objects.create(
-                    upload_session=instance,
-                    filename=uploaded_file.name,
-                    staged_file_id=uploaded_file.uuid,
-                )
+        instance = super().save(commit=False)
+
+        # Create links between the created session and all uploaded files
+        uploaded_files = self.cleaned_data[
+            "files"]  # type: List[StagedAjaxFile]
+
+        raw_files = [
+            RawImageFile(
+                upload_session=instance,
+                filename=uploaded_file.name,
+                staged_file_id=uploaded_file.uuid,
+            ) for uploaded_file in uploaded_files
+        ]
+
+        if commit:
+            with transaction.atomic():
+                instance.save()
+                RawImageFile.objects.bulk_create(raw_files)
 
         return instance
 
diff --git a/app/grandchallenge/cases/signals.py b/app/grandchallenge/cases/signals.py
--- a/app/grandchallenge/cases/signals.py
+++ b/app/grandchallenge/cases/signals.py
@@ -11,17 +11,20 @@
 @receiver(post_save, sender=RawImageUploadSession)
 @disable_for_loaddata
 def queue_build_image_job(
-        instance: RawImageUploadSession=None, created: bool=False,
-        *_, **__):
+        instance: RawImageUploadSession = None, created: bool = False, *_, **__
+):
     if created:
         try:
-            task = build_images.apply_async(
-                args=(instance.pk,),
-                countdown=5, # Wait a bit - immediate start won't work!
+
+            RawImageUploadSession.objects.filter(pk=instance.pk).update(
+                session_state=UPLOAD_SESSION_STATE.queued,
+                processing_task=instance.pk
             )
-            instance.session_state = UPLOAD_SESSION_STATE.queued
-            instance.processing_task = task.id
-            instance.save()
+
+            build_images.apply_async(
+                task_id=str(instance.pk), args=(instance.pk,),
+            )
+
         except Exception as e:
             instance.session_state = UPLOAD_SESSION_STATE.stopped
             instance.error_message = f"Could not start job: {e}"
diff --git a/app/grandchallenge/cases/tasks.py b/app/grandchallenge/cases/tasks.py
--- a/app/grandchallenge/cases/tasks.py
+++ b/app/grandchallenge/cases/tasks.py
@@ -146,12 +146,7 @@ def remove_duplicate_files(session_files: Sequence[RawImageFile]) -> Tuple[
     )
 
 
-@shared_task(
-    autoretry_for=(RawImageUploadSession.DoesNotExist,),
-    default_retry_delay=60,
-    retry_kwargs={
-        'max_retries': 15
-    })
+@shared_task
 def build_images(upload_session_uuid: UUID):
     """
     Task which analyzes an upload session and attempts to extract and store


Test patch:
diff --git a/app/tests/cases_tests/job_test_utils.py b/app/tests/cases_tests/job_test_utils.py
deleted file mode 100644
--- a/app/tests/cases_tests/job_test_utils.py
+++ /dev/null
@@ -1,88 +0,0 @@
-import contextlib
-import uuid
-
-
-@contextlib.contextmanager
-def replace_var(module, value_name, substitute):
-    old_value = getattr(module, value_name)
-    try:
-        setattr(module, value_name, substitute)
-        yield
-    finally:
-        setattr(module, value_name, old_value)
-
-
-
-class CeleryTaskCollectorTask:
-    """
-    Celery mock task used by CeleryTaskCollector to mimic celery.
-    """
-    def __init__(self, args=(), kwargs={}, countdown=None, eta=None, expires=None):
-        self.__uuid = uuid.uuid4()
-        self.args = args
-        self.kwargs = kwargs
-
-        self.countdown = countdown
-        self.eta = eta
-        self.expires = expires
-
-    @property
-    def id(self):
-        return self.__uuid
-
-
-class CeleryTaskCollector:
-    """
-    This class provides a dummy celery interface that can be used to
-    replace celery shared_task functions. Calls to call_async are collected
-    and can later be iterated over using the iteration function of this
-    queue.
-
-    Parameters
-    ----------
-    fun:
-        The function, specifically, a celery shared_task that should be
-        overridden.
-    """
-    def __init__(self, fun):
-        self.__fun = fun
-        self.__calls = []
-
-    def __call__(self, *args, **kwargs):
-        """
-        Support of direct calls
-        """
-        return self.__fun(*args, **kwargs)
-
-    def apply_async(self, args=(), kwargs={}, countdown=None, eta=None, expires=None):
-        """
-        Emulation of apply_async of a shared_job in celery.
-
-        Parameters
-        ----------
-        args
-        kwargs
-        countdown
-        eta
-        expires
-
-        Returns
-        -------
-
-        """
-        task = CeleryTaskCollectorTask(
-            args,
-            kwargs,
-            countdown=countdown,
-            eta=eta,
-            expires=expires)
-        self.__calls.append(task)
-        return task
-
-    def execute_calls(self):
-        """
-        Consumes all queued tasks and executes them on the main thread.
-        """
-        while self.__calls:
-            call = self.__calls.pop()
-            self(*call.args, **call.kwargs)
diff --git a/app/tests/cases_tests/test_background_tasks.py b/app/tests/cases_tests/test_background_tasks.py
--- a/app/tests/cases_tests/test_background_tasks.py
+++ b/app/tests/cases_tests/test_background_tasks.py
@@ -2,21 +2,20 @@
 
 import pytest
 
-from grandchallenge.cases import signals
-from grandchallenge.cases.models import RawImageFile, RawImageUploadSession, \
-    UPLOAD_SESSION_STATE, Image
-from grandchallenge.cases.tasks import build_images
+from grandchallenge.cases.models import (
+    RawImageFile, RawImageUploadSession, UPLOAD_SESSION_STATE, Image
+)
 from grandchallenge.jqfileupload.widgets.uploader import StagedAjaxFile
 from tests.cases_tests import RESOURCE_PATH
-from tests.cases_tests.job_test_utils import CeleryTaskCollector, replace_var
 from tests.jqfileupload_tests.external_test_support import \
     create_file_from_filepath
 
 
-
 def create_raw_upload_image_session(
-        images: List[str]) -> Tuple[RawImageUploadSession, Dict[str, RawImageFile]]:
-    upload_session = RawImageUploadSession.objects.create()
+        images: List[str], delete_file=False,
+) -> Tuple[RawImageUploadSession, Dict[str, RawImageFile]]:
+    upload_session = RawImageUploadSession()
+
     uploaded_images = {}
     for image in images:
         staged_file = create_file_from_filepath(RESOURCE_PATH / image)
@@ -26,6 +25,13 @@ def create_raw_upload_image_session(
             staged_file_id=staged_file.uuid,
         )
         uploaded_images[staged_file.name] = image
+
+    if delete_file:
+        StagedAjaxFile(
+            uploaded_images["image10x10x10.zraw"].staged_file_id).delete()
+
+    upload_session.save()
+
     return upload_session, uploaded_images
 
 
@@ -44,135 +50,151 @@ def test_file_session_creation():
 
 
 @pytest.mark.django_db
-def test_mhd_file_creation():
-    task_collector = CeleryTaskCollector(signals.build_images)
-    with replace_var(signals, "build_images", task_collector):
-        images = [
-            "image10x10x10.zraw",
-            "image10x10x10.mhd",
-            "image10x10x10.mha",
-            "image10x10x10-extra-stuff.mhd",
-            "invalid_utf8.mhd",
-            "no_image",
-        ]
-        session, uploaded_images = create_raw_upload_image_session(images)
-        task_collector.execute_calls()
-
-        session.refresh_from_db()
-        assert session.session_state == UPLOAD_SESSION_STATE.stopped
-        assert session.error_message is None
-
-        assert Image.objects.filter(origin=session).count() == 3
-
-        for name, db_object in uploaded_images.items():
-            name: str
-            db_object: RawImageFile
-
-            db_object.refresh_from_db()
-
-            assert db_object.staged_file_id is None
-            if name in ("no_image", "invalid_utf8.mhd"):
-                assert db_object.error is not None
-            else:
-                assert db_object.error is None
+def test_mhd_file_creation(settings):
+    # Override the celery settings
+    settings.task_eager_propagates = True,
+    settings.task_always_eager = True,
+    settings.broker_url = 'memory://',
+    settings.backend = 'memory'
+
+    # with replace_var(signals, "build_images", task_collector):
+    images = [
+        "image10x10x10.zraw",
+        "image10x10x10.mhd",
+        "image10x10x10.mha",
+        "image10x10x10-extra-stuff.mhd",
+        "invalid_utf8.mhd",
+        "no_image",
+    ]
+    session, uploaded_images = create_raw_upload_image_session(images)
+
+    session.refresh_from_db()
+    assert session.session_state == UPLOAD_SESSION_STATE.stopped
+    assert session.error_message is None
+
+    assert Image.objects.filter(origin=session).count() == 3
+
+    for name, db_object in uploaded_images.items():
+        name: str
+        db_object: RawImageFile
+
+        db_object.refresh_from_db()
+
+        assert db_object.staged_file_id is None
+        if name in ("no_image", "invalid_utf8.mhd"):
+            assert db_object.error is not None
+        else:
+            assert db_object.error is None
 
 
 @pytest.mark.django_db
-def test_staged_uploaded_file_cleanup_interferes_with_image_build():
-    task_collector = CeleryTaskCollector(signals.build_images)
-    with replace_var(signals, "build_images", task_collector):
-        images = [
-            "image10x10x10.zraw",
-            "image10x10x10.mhd",
-        ]
-        session, uploaded_images = create_raw_upload_image_session(images)
-        StagedAjaxFile(uploaded_images["image10x10x10.zraw"].staged_file_id).delete()
-        task_collector.execute_calls()
-
-        session.refresh_from_db()
-        assert session.session_state == UPLOAD_SESSION_STATE.stopped
-        assert session.error_message is not None
+def test_staged_uploaded_file_cleanup_interferes_with_image_build(settings):
+    # Override the celery settings
+    settings.task_eager_propagates = True,
+    settings.task_always_eager = True,
+    settings.broker_url = 'memory://',
+    settings.backend = 'memory'
+
+    images = [
+        "image10x10x10.zraw",
+        "image10x10x10.mhd",
+    ]
+    session, uploaded_images = create_raw_upload_image_session(
+        images, delete_file=True
+    )
+
+    session.refresh_from_db()
+    assert session.session_state == UPLOAD_SESSION_STATE.stopped
+    assert session.error_message is not None
 
 
 @pytest.mark.django_db
-def test_no_convertible_file():
-    task_collector = CeleryTaskCollector(signals.build_images)
-    with replace_var(signals, "build_images", task_collector):
-        images = [
-            "no_image",
-            "image10x10x10.mhd",
-            "referring_to_system_file.mhd",
-        ]
-        session, uploaded_images = create_raw_upload_image_session(images)
-        task_collector.execute_calls()
-
-        session.refresh_from_db()
-        assert session.session_state == UPLOAD_SESSION_STATE.stopped
-        assert session.error_message is None
-
-        no_image_image = list(uploaded_images.values())[0]
-        no_image_image.refresh_from_db()
-        assert no_image_image.error is not None
-
-        lonely_mhd_image = list(uploaded_images.values())[1]
-        lonely_mhd_image.refresh_from_db()
-        assert lonely_mhd_image.error is not None
-
-        sys_file_image = list(uploaded_images.values())[2]
-        sys_file_image.refresh_from_db()
-        assert sys_file_image.error is not None
+def test_no_convertible_file(settings):
+    # Override the celery settings
+    settings.task_eager_propagates = True,
+    settings.task_always_eager = True,
+    settings.broker_url = 'memory://',
+    settings.backend = 'memory'
+
+    images = [
+        "no_image",
+        "image10x10x10.mhd",
+        "referring_to_system_file.mhd",
+    ]
+    session, uploaded_images = create_raw_upload_image_session(images)
+
+    session.refresh_from_db()
+    assert session.session_state == UPLOAD_SESSION_STATE.stopped
+    assert session.error_message is None
+
+    no_image_image = list(uploaded_images.values())[0]
+    no_image_image.refresh_from_db()
+    assert no_image_image.error is not None
+
+    lonely_mhd_image = list(uploaded_images.values())[1]
+    lonely_mhd_image.refresh_from_db()
+    assert lonely_mhd_image.error is not None
+
+    sys_file_image = list(uploaded_images.values())[2]
+    sys_file_image.refresh_from_db()
+    assert sys_file_image.error is not None
 
 
 @pytest.mark.django_db
-def test_errors_on_files_with_duplicate_file_names():
-    task_collector = CeleryTaskCollector(signals.build_images)
-    with replace_var(signals, "build_images", task_collector):
-        images = [
-            "image10x10x10.zraw",
-            "image10x10x10.mhd",
-            "image10x10x10.zraw",
-            "image10x10x10.mhd",
-        ]
-        session, uploaded_images = create_raw_upload_image_session(images)
-        uploaded_images = RawImageFile.objects.filter(upload_session=session).all()
-        assert len(uploaded_images) == 4
-
-        task_collector.execute_calls()
-
-        session.refresh_from_db()
-        assert session.session_state == UPLOAD_SESSION_STATE.stopped
-        assert session.error_message is None
-
-        for raw_image in uploaded_images:
-            raw_image.refresh_from_db()
-            assert raw_image.error is not None
+def test_errors_on_files_with_duplicate_file_names(settings):
+    # Override the celery settings
+    settings.task_eager_propagates = True,
+    settings.task_always_eager = True,
+    settings.broker_url = 'memory://',
+    settings.backend = 'memory'
+
+    images = [
+        "image10x10x10.zraw",
+        "image10x10x10.mhd",
+        "image10x10x10.zraw",
+        "image10x10x10.mhd",
+    ]
+    session, uploaded_images = create_raw_upload_image_session(images)
+    uploaded_images = RawImageFile.objects.filter(
+        upload_session=session).all()
+    assert len(uploaded_images) == 4
+
+    session.refresh_from_db()
+    assert session.session_state == UPLOAD_SESSION_STATE.stopped
+    assert session.error_message is None
+
+    for raw_image in uploaded_images:
+        raw_image.refresh_from_db()
+        assert raw_image.error is not None
 
 
 @pytest.mark.django_db
-def test_mhd_file_annotation_creation():
-    task_collector = CeleryTaskCollector(signals.build_images)
-    with replace_var(signals, "build_images", task_collector):
-        images = [
-            "image5x6x7.mhd",
-            "image5x6x7.zraw",
-        ]
-        session, uploaded_images = create_raw_upload_image_session(images)
-
-        task_collector.execute_calls()
-
-        session.refresh_from_db()
-        assert session.session_state == UPLOAD_SESSION_STATE.stopped
-        assert session.error_message is None
-
-        images = Image.objects.filter(origin=session).all()
-        assert len(images) == 1
-
-        raw_image_file = list(uploaded_images.values())[0]
-        raw_image_file: RawImageFile
-        raw_image_file.refresh_from_db()
-        assert raw_image_file.staged_file_id is None
-
-        image = images[0]
-        assert image.shape == [5, 6, 7]
-        assert image.shape_without_color == [5, 6, 7]
-        assert image.color_space == Image.COLOR_SPACE_GRAY
+def test_mhd_file_annotation_creation(settings):
+    # Override the celery settings
+    settings.task_eager_propagates = True,
+    settings.task_always_eager = True,
+    settings.broker_url = 'memory://',
+    settings.backend = 'memory'
+
+    images = [
+        "image5x6x7.mhd",
+        "image5x6x7.zraw",
+    ]
+    session, uploaded_images = create_raw_upload_image_session(images)
+
+    session.refresh_from_db()
+    assert session.session_state == UPLOAD_SESSION_STATE.stopped
+    assert session.error_message is None
+
+    images = Image.objects.filter(origin=session).all()
+    assert len(images) == 1
+
+    raw_image_file = list(uploaded_images.values())[0]
+    raw_image_file: RawImageFile
+    raw_image_file.refresh_from_db()
+    assert raw_image_file.staged_file_id is None
+
+    image = images[0]
+    assert image.shape == [5, 6, 7]
+    assert image.shape_without_color == [5, 6, 7]
+    assert image.color_space == Image.COLOR_SPACE_GRAY
diff --git a/app/tests/cases_tests/test_forms.py b/app/tests/cases_tests/test_forms.py
--- a/app/tests/cases_tests/test_forms.py
+++ b/app/tests/cases_tests/test_forms.py
@@ -4,10 +4,8 @@
 import pytest
 from django.test import Client
 
-from grandchallenge.cases import signals
 from grandchallenge.cases.models import RawImageUploadSession
 from tests.cases_tests import RESOURCE_PATH
-from tests.cases_tests.job_test_utils import CeleryTaskCollector, replace_var
 from tests.factories import SUPER_SECURE_TEST_PASSWORD, UserFactory
 from tests.jqfileupload_tests.external_test_support import (
     create_file_from_filepath
@@ -15,40 +13,43 @@
 
 
 @pytest.mark.django_db
-def test_upload_some_images(client: Client, ChallengeSet):
-    task_collector = CeleryTaskCollector(signals.build_images)
-    with replace_var(signals, "build_images", task_collector):
-
-        response = client.get("/cases/upload/")
-        assert response.status_code != 200
-
-        staff_user = UserFactory(is_staff=True)
-
-        assert client.login(
-            username=staff_user.username, password=SUPER_SECURE_TEST_PASSWORD
-        )
-
-        response = client.get("/cases/upload/")
-        assert response.status_code == 200
-
-        file1 = create_file_from_filepath(RESOURCE_PATH / "image10x10x10.mha")
-
-        response = client.post(
-            "/cases/upload/",
-            data={
-                "files": f"{file1.uuid}",
-            }
-        )
-        assert response.status_code == 302
-
-        redirect_match = re.match(
-            r"/cases/uploaded/(?P<uuid>[^/]+)/?$",
-            response["Location"])
-
-        assert redirect_match is not None
-        assert RawImageUploadSession.objects.filter(
-            pk=UUID(redirect_match.group("uuid"))
-        ).exists()
-
-        response = client.get(response["Location"])
-        assert response.status_code == 200
+def test_upload_some_images(client: Client, ChallengeSet, settings):
+    # Override the celery settings
+    settings.task_eager_propagates = True,
+    settings.task_always_eager = True,
+    settings.broker_url = 'memory://',
+    settings.backend = 'memory'
+
+    response = client.get("/cases/upload/")
+    assert response.status_code != 200
+
+    staff_user = UserFactory(is_staff=True)
+
+    assert client.login(
+        username=staff_user.username, password=SUPER_SECURE_TEST_PASSWORD
+    )
+
+    response = client.get("/cases/upload/")
+    assert response.status_code == 200
+
+    file1 = create_file_from_filepath(RESOURCE_PATH / "image10x10x10.mha")
+
+    response = client.post(
+        "/cases/upload/",
+        data={
+            "files": f"{file1.uuid}",
+        }
+    )
+    assert response.status_code == 302
+
+    redirect_match = re.match(
+        r"/cases/uploaded/(?P<uuid>[^/]+)/?$",
+        response["Location"])
+
+    assert redirect_match is not None
+    assert RawImageUploadSession.objects.filter(
+        pk=UUID(redirect_match.group("uuid"))
+    ).exists()
+
+    response = client.get(response["Location"])
+    assert response.status_code == 200