Repository: comic/grand-challenge.org

Issue:
Add a warning about max submissions reached for challenge admins



Discussion:

Resolving patch:
diff --git a/app/grandchallenge/evaluation/migrations/0020_phase_public.py b/app/grandchallenge/evaluation/migrations/0020_phase_public.py
--- a/app/grandchallenge/evaluation/migrations/0020_phase_public.py
+++ b/app/grandchallenge/evaluation/migrations/0020_phase_public.py
@@ -15,7 +15,7 @@ class Migration(migrations.Migration):
             name="public",
             field=models.BooleanField(
                 default=True,
-                help_text="Uncheck this box to hide this phase's submission page and leaderboard from participants. Participants will then no longer have access to their previous submissions and evaluations from this phase if they exist, and they will no longer see the respective submit and leaderboard tabs for this phase. For you as admin these tabs remain visible.Note that hiding a phase is only possible if submissions for this phase are closed for participants.",
+                help_text="Uncheck this box to hide this phase's submission page and leaderboard from participants. Participants will then no longer have access to their previous submissions and evaluations from this phase if they exist, and they will no longer see the respective submit and leaderboard tabs for this phase. For you as admin these tabs remain visible. Note that hiding a phase is only possible if submissions for this phase are closed for participants.",
             ),
         ),
     ]
diff --git a/app/grandchallenge/evaluation/migrations/0031_phase_total_number_of_submissions_allowed.py b/app/grandchallenge/evaluation/migrations/0031_phase_total_number_of_submissions_allowed.py
--- a/app/grandchallenge/evaluation/migrations/0031_phase_total_number_of_submissions_allowed.py
+++ b/app/grandchallenge/evaluation/migrations/0031_phase_total_number_of_submissions_allowed.py
@@ -18,7 +18,7 @@ class Migration(migrations.Migration):
             name="total_number_of_submissions_allowed",
             field=models.PositiveSmallIntegerField(
                 blank=True,
-                help_text="Total number of submissions allowed for this phase for all users together.",
+                help_text="Total number of successful submissions allowed for this phase for all users together.",
                 null=True,
             ),
         ),
diff --git a/app/grandchallenge/evaluation/models.py b/app/grandchallenge/evaluation/models.py
--- a/app/grandchallenge/evaluation/models.py
+++ b/app/grandchallenge/evaluation/models.py
@@ -425,7 +425,7 @@ class Phase(UUIDModel, ViewContentMixin):
             "have access to their previous submissions and evaluations from this "
             "phase if they exist, and they will no longer see the "
             "respective submit and leaderboard tabs for this phase. "
-            "For you as admin these tabs remain visible."
+            "For you as admin these tabs remain visible. "
             "Note that hiding a phase is only possible if submissions for "
             "this phase are closed for participants."
         ),
@@ -451,7 +451,7 @@ class Phase(UUIDModel, ViewContentMixin):
     total_number_of_submissions_allowed = models.PositiveSmallIntegerField(
         blank=True,
         null=True,
-        help_text="Total number of submissions allowed for this phase for all users together.",
+        help_text="Total number of successful submissions allowed for this phase for all users together.",
     )
 
     class Meta:
@@ -657,7 +657,8 @@ def submission_period_is_open_now(self):
     @property
     def open_for_submissions(self):
         return (
-            self.submission_period_is_open_now
+            self.public
+            and self.submission_period_is_open_now
             and self.submissions_limit_per_user_per_period > 0
             and not self.exceeds_total_number_of_submissions_allowed
         )
@@ -669,18 +670,32 @@ def exceeds_total_number_of_submissions_allowed(self):
             and self.percent_of_total_submissions_allowed >= 100
         )
 
-    @property
+    @cached_property
     def percent_of_total_submissions_allowed(self):
-        if self.total_number_of_submissions_allowed:
+        if self.total_number_of_submissions_allowed is None:
+            return None
+        elif self.total_number_of_submissions_allowed == 0:
+            return 100
+        else:
+            # Allow all submissions by challenge admins
+            # and do not count cancellations or failures
             return round(
                 (
-                    self.submission_set.count()
+                    self.submission_set.exclude(
+                        evaluation__status__in=[
+                            Evaluation.FAILURE,
+                            Evaluation.CANCELLED,
+                        ]
+                    )
+                    .exclude(
+                        creator__in=self.challenge.admins_group.user_set.all()
+                    )
+                    .distinct()
+                    .count()
                     / self.total_number_of_submissions_allowed
                 )
                 * 100
             )
-        else:
-            return None
 
     @property
     def status(self):
@@ -718,13 +733,6 @@ def submission_status_string(self):
         else:
             raise NotImplementedError(f"{self.status} not implemented")
 
-    @property
-    def inconsistent_submission_information(self):
-        return (
-            self.submissions_limit_per_user_per_period == 0
-            and self.submission_period_is_open_now
-        )
-
     @cached_property
     def active_image(self):
         """


Test patch:
diff --git a/app/tests/challenges_tests/test_models.py b/app/tests/challenges_tests/test_models.py
--- a/app/tests/challenges_tests/test_models.py
+++ b/app/tests/challenges_tests/test_models.py
@@ -6,8 +6,9 @@
 from machina.apps.forum_conversation.models import Topic
 
 from grandchallenge.challenges.models import Challenge
+from grandchallenge.evaluation.models import Evaluation
 from grandchallenge.notifications.models import Notification
-from tests.evaluation_tests.factories import PhaseFactory, SubmissionFactory
+from tests.evaluation_tests.factories import EvaluationFactory, PhaseFactory
 from tests.factories import ChallengeFactory, UserFactory
 from tests.notifications_tests.factories import TopicFactory
 
@@ -100,10 +101,31 @@ def test_submission_limit_status():
         4, total_number_of_submissions_allowed=10
     )
     p5 = PhaseFactory()
-    SubmissionFactory.create_batch(10, phase=p1)
-    SubmissionFactory.create_batch(4, phase=p2)
-    SubmissionFactory.create_batch(8, phase=p3)
-    SubmissionFactory.create_batch(12, phase=p4)
+
+    admin = UserFactory()
+    p3.challenge.add_admin(user=admin)
+
+    EvaluationFactory.create_batch(
+        10, submission__phase=p1, status=Evaluation.SUCCESS
+    )
+    EvaluationFactory.create_batch(
+        4, submission__phase=p2, status=Evaluation.SUCCESS
+    )
+    EvaluationFactory.create_batch(
+        10, submission__phase=p2, status=Evaluation.FAILURE
+    )
+    EvaluationFactory.create_batch(
+        8, submission__phase=p3, status=Evaluation.SUCCESS
+    )
+    EvaluationFactory.create_batch(
+        8,
+        submission__phase=p3,
+        submission__creator=admin,
+        status=Evaluation.SUCCESS,
+    )
+    EvaluationFactory.create_batch(
+        12, submission__phase=p4, status=Evaluation.SUCCESS
+    )
 
     assert p1.percent_of_total_submissions_allowed == 100
     assert p2.percent_of_total_submissions_allowed == 40