Repository: comic/grand-challenge.org

Issue:
Possible to submit algorithm to Phase where inputs/outputs do not match
Results in an infinitely executing submission.


Discussion:
The development tests work OK but this appears to happen for SynthRad. Phases:

- MRI Image (Image), Body (Segmentation), and Region (String) and the following outputs: Synthetic CT image (Image)
- CBCT image (Image), Body (Segmentation), and Region (String) and the following outputs: Synthetic CT image (Image)

I can replicate this in development.

Resolving patch:
diff --git a/app/grandchallenge/algorithms/forms.py b/app/grandchallenge/algorithms/forms.py
--- a/app/grandchallenge/algorithms/forms.py
+++ b/app/grandchallenge/algorithms/forms.py
@@ -17,7 +17,7 @@
 from django.core.exceptions import ObjectDoesNotExist, ValidationError
 from django.core.files.base import ContentFile
 from django.core.validators import RegexValidator
-from django.db.models import Count
+from django.db.models import Count, Q
 from django.db.transaction import on_commit
 from django.forms import (
     CharField,
@@ -307,47 +307,41 @@ def get_phase_algorithm_inputs_outputs(self):
         )
 
     @cached_property
-    def get_user_algorithms_for_phase(self):
+    def user_algorithms_for_phase(self):
         inputs, outputs = self.get_phase_algorithm_inputs_outputs()
         return (
             get_objects_for_user(self._user, "algorithms.change_algorithm")
             .annotate(
-                input_count=Count("inputs", distinct=True),
-                output_count=Count("outputs", distinct=True),
+                total_input_count=Count("inputs", distinct=True),
+                total_output_count=Count("outputs", distinct=True),
+                relevant_input_count=Count(
+                    "inputs", filter=Q(inputs__in=inputs), distinct=True
+                ),
+                relevant_output_count=Count(
+                    "outputs", filter=Q(outputs__in=outputs), distinct=True
+                ),
             )
             .filter(
-                inputs__in=inputs,
-                outputs__in=outputs,
-                input_count=len(inputs),
-                output_count=len(outputs),
+                total_input_count=len(inputs),
+                total_output_count=len(outputs),
+                relevant_input_count=len(inputs),
+                relevant_output_count=len(outputs),
             )
         )
 
     @cached_property
-    def get_user_active_images_for_phase(self):
-        inputs, outputs = self.get_phase_algorithm_inputs_outputs()
-        return (
-            get_objects_for_user(
-                user=self._user,
-                perms="algorithms.change_algorithmimage",
-                klass=AlgorithmImage.objects.active_images(),
-            )
+    def user_active_images_for_phase(self):
+        return get_objects_for_user(
+            user=self._user,
+            perms="algorithms.change_algorithmimage",
+            klass=AlgorithmImage.objects.active_images()
             .select_related("algorithm")
-            .annotate(
-                input_count=Count("algorithm__inputs", distinct=True),
-                output_count=Count("algorithm__outputs", distinct=True),
-            )
-            .filter(
-                algorithm__inputs__in=inputs,
-                algorithm__outputs__in=outputs,
-                input_count=len(inputs),
-                output_count=len(outputs),
-            )
+            .filter(algorithm__in=self.user_algorithms_for_phase),
         )
 
     @cached_property
     def user_algorithm_count(self):
-        return self.get_user_algorithms_for_phase.count()
+        return self.user_algorithms_for_phase.count()
 
 
 class AlgorithmForPhaseForm(UserAlgorithmsForPhaseMixin, ModelForm):
diff --git a/app/grandchallenge/evaluation/forms.py b/app/grandchallenge/evaluation/forms.py
--- a/app/grandchallenge/evaluation/forms.py
+++ b/app/grandchallenge/evaluation/forms.py
@@ -321,7 +321,7 @@ def __init__(self, *args, user, phase: Phase, **kwargs):  # noqa: C901
 
             self.fields[
                 "algorithm_image"
-            ].queryset = self.get_user_active_images_for_phase.order_by(
+            ].queryset = self.user_active_images_for_phase.order_by(
                 "algorithm__title"
             )
 
diff --git a/app/grandchallenge/evaluation/views/__init__.py b/app/grandchallenge/evaluation/views/__init__.py
--- a/app/grandchallenge/evaluation/views/__init__.py
+++ b/app/grandchallenge/evaluation/views/__init__.py
@@ -795,7 +795,7 @@ def get_context_data(self, **kwargs):
         context.update(
             {
                 "user_algorithm_count": form.user_algorithm_count,
-                "user_algorithms": form.get_user_algorithms_for_phase,
+                "user_algorithms": form.user_algorithms_for_phase,
                 "max_num_algorithms": settings.ALGORITHMS_MAX_NUMBER_PER_USER_PER_PHASE,
                 "hide_form": self.hide_form(form=form),
             }


Test patch:
diff --git a/app/tests/evaluation_tests/test_views.py b/app/tests/evaluation_tests/test_views.py
--- a/app/tests/evaluation_tests/test_views.py
+++ b/app/tests/evaluation_tests/test_views.py
@@ -833,14 +833,7 @@ def test_create_algorithm_for_phase_limits(client):
     assert "You have created 2 out of 3 possible algorithms" in str(
         response.content
     )
-    for alg in [alg1, alg4]:
-        assert f'<a href="{alg.get_absolute_url()}#containers"' in str(
-            response.content
-        )
-    for alg in [alg2, alg3, alg5, alg6]:
-        assert f'<a href="{alg.get_absolute_url()}#containers"' not in str(
-            response.content
-        )
+    assert {*response.context["user_algorithms"]} == {alg1, alg4}
 
     # clicking on confirm will show the form
     response = get_view_for_user(
@@ -870,14 +863,7 @@ def test_create_algorithm_for_phase_limits(client):
         "You have created the maximum number of allowed algorithms for this phase!"
         in str(response.content)
     )
-    for alg in [alg1, alg2, alg3]:
-        assert f'<a href="{alg.get_absolute_url()}#containers"' in str(
-            response.content
-        )
-    for alg in [alg4, alg5, alg6]:
-        assert f'<a href="{alg.get_absolute_url()}#containers"' not in str(
-            response.content
-        )
+    assert {*response.context["user_algorithms"]} == {alg1, alg2, alg3}
 
     # force submitting a form with data for a user that has reached the limit,
     # will not work, they will just get redirected to the page telling them that they
@@ -901,14 +887,7 @@ def test_create_algorithm_for_phase_limits(client):
         "You have created the maximum number of allowed algorithms for this phase!"
         in str(response.content)
     )
-    for alg in [alg1, alg2, alg3]:
-        assert f'<a href="{alg.get_absolute_url()}#containers"' in str(
-            response.content
-        )
-    for alg in [alg4, alg5, alg6]:
-        assert f'<a href="{alg.get_absolute_url()}#containers"' not in str(
-            response.content
-        )
+    assert {*response.context["user_algorithms"]} == {alg1, alg2, alg3}
     assert not Algorithm.objects.filter(title="foo").exists()