Repository: comic/grand-challenge.org

Issue:
Uploading a non-zip or non-text file to an evaluation does not change the job state from queued to failed
Steps to reproduce:

1. Upload a container image as a method for the demo evaluation phase, wait until it is ready to be used
2. Submit a `.7z` file as the submission
3. The evaluation created will remain in the `Queued` state forever, it should be marked as failed

This occurs https://github.com/comic/grand-challenge.org/blob/a56fee78f605e957880a428c1698a1613af985ca/app/grandchallenge/evaluation/models.py#L611-L613

The associated evaluation should be marked as failed here (use `evaluation.update_status(status=Evaluation.FAILURE, stderr=f"{mimetype} files are not supported.")`). 

Whilst here, it would be great to solve this TODO: https://github.com/comic/grand-challenge.org/blob/a56fee78f605e957880a428c1698a1613af985ca/app/grandchallenge/evaluation/models.py#L589-L591



Discussion:

Resolving patch:
diff --git a/app/grandchallenge/evaluation/emails.py b/app/grandchallenge/evaluation/emails.py
--- a/app/grandchallenge/evaluation/emails.py
+++ b/app/grandchallenge/evaluation/emails.py
@@ -1,37 +1,45 @@
 from django.conf import settings
 from django.contrib.sites.models import Site
 from django.core.mail import send_mail
+from django.utils.html import format_html
 
 from grandchallenge.subdomains.utils import reverse
 
 
 def send_failed_evaluation_email(evaluation):
     site = Site.objects.get_current()
-    message = (
-        f"Dear {{}},\n\n"
-        f"Unfortunately the evaluation for the submission to "
-        f"{evaluation.submission.phase.challenge.short_name} failed with an "
-        f"error. The error message is:\n\n"
-        f"{evaluation.error_message}\n\n"
-        f"You may wish to try and correct this, or contact the challenge "
-        f"organizers. The following information may help them:\n"
-        f"User: {evaluation.submission.creator.username}\n"
-        f"Evaluation ID: {evaluation.pk}\n"
-        f"Submission ID: {evaluation.submission.pk}\n\n"
-        f"Regards,\n"
-        f"{site.name}\n\n"
-        f"This is an automated service email from {site.domain}."
-    )
     recipients = list(evaluation.submission.phase.challenge.get_admins())
     recipients.append(evaluation.submission.creator)
     for recipient in recipients:
+        message = format_html(
+            "Dear {recipient},\n\n"
+            "Unfortunately the evaluation for the submission to {challenge} "
+            "failed with an error. The error message is:\n\n"
+            "{error_message}\n\n"
+            "You may wish to try and correct this, or contact the challenge "
+            "organizers. The following information may help them:\n"
+            "User: {user}\n"
+            "Evaluation ID: {evaluation_id}\n"
+            "Submission ID: {submission_id}\n\n"
+            "Regards,\n"
+            "{site}\n\n"
+            "This is an automated service email from {domain}",
+            recipient=recipient.username,
+            challenge=evaluation.submission.phase.challenge.short_name,
+            error_message=evaluation.error_message,
+            user=evaluation.submission.creator.username,
+            evaluation_id=evaluation.pk,
+            submission_id=evaluation.submission.pk,
+            site=site.name,
+            domain=site.domain,
+        )
         send_mail(
             subject=(
                 f"[{site.domain.lower()}] "
                 f"[{evaluation.submission.phase.challenge.short_name.lower()}] "
                 f"Evaluation Failed"
             ),
-            message=message.format(recipient.username),
+            message=message,
             from_email=settings.DEFAULT_FROM_EMAIL,
             recipient_list=[recipient.email],
         )
@@ -42,43 +50,83 @@ def send_successful_evaluation_email(evaluation):
     challenge = evaluation.submission.phase.challenge
 
     recipients = list(challenge.get_admins())
-    message = (
-        f"Dear {{}},\n\n"
-        f"There is a new result for {challenge.short_name} from "
-        f"{evaluation.submission.creator.username}. "
-    )
+
     if evaluation.published:
-        leaderboard_url = reverse(
-            "evaluation:leaderboard",
-            kwargs={
-                "challenge_short_name": challenge.short_name,
-                "slug": evaluation.submission.phase.slug,
-            },
+        recipients.append(evaluation.submission.creator)
+
+    for recipient in recipients:
+        message = format_html(
+            "Dear {recipient},\n\n"
+            "There is a new result for {challenge} from {user}.",
+            recipient=recipient.username,
+            challenge=challenge.short_name,
+            user=evaluation.submission.creator.username,
         )
-        message += (
-            f"You can view the result on the leaderboard here: "
-            f"{leaderboard_url}"
+        if evaluation.published:
+            leaderboard_url = reverse(
+                "evaluation:leaderboard",
+                kwargs={
+                    "challenge_short_name": challenge.short_name,
+                    "slug": evaluation.submission.phase.slug,
+                },
+            )
+            message += format_html(
+                "You can view the result on the leaderboard here: {leaderboard}\n\n",
+                leaderboard=leaderboard_url,
+            )
+        else:
+            message += format_html(
+                "You can publish the result on the leaderboard here: {leaderboard}\n\n",
+                leaderboard=evaluation.get_absolute_url(),
+            )
+        message += format_html(
+            "Regards,\n"
+            "{site}\n\n"
+            "This is an automated service email from {domain}",
+            site=site.name,
+            domain=site.domain,
         )
-        recipients.append(evaluation.submission.creator)
-    else:
-        message += (
-            f"You can publish the result on the leaderboard here: "
-            f"{evaluation.get_absolute_url()}"
+        send_mail(
+            subject=(
+                f"[{site.domain.lower()}] "
+                f"[{challenge.short_name.lower()}] "
+                f"New Result"
+            ),
+            message=message,
+            from_email=settings.DEFAULT_FROM_EMAIL,
+            recipient_list=[recipient.email],
         )
-    message += (
-        f"\n\n"
-        f"Regards,\n"
-        f"{site.name}\n\n"
-        f"This is an automated service email from {site.domain}."
-    )
+
+
+def send_missing_method_email(submission):
+    site = Site.objects.get_current()
+    challenge = submission.phase.challenge
+
+    recipients = challenge.get_admins()
+
     for recipient in recipients:
+        message = format_html(
+            "Dear {recipient},\n\n"
+            "The submission from {user} could not be evaluated, "
+            "because there is no valid evaluation method for {challenge} in the {phase} evaluation phase. "
+            "Please upload an evaluation method container.\n\n"
+            "Regards,\n"
+            "{site}\n\n"
+            "This is an automated service email from {domain}.",
+            recipient=recipient.username,
+            user=submission.creator.username,
+            challenge=challenge.short_name,
+            phase=submission.phase.slug,
+            site=site.name,
+            domain=site.domain,
+        )
         send_mail(
             subject=(
                 f"[{site.domain.lower()}] "
                 f"[{challenge.short_name.lower()}] "
-                f"New Result"
+                f"No evaluation method"
             ),
-            message=message.format(recipient.username),
+            message=message,
             from_email=settings.DEFAULT_FROM_EMAIL,
             recipient_list=[recipient.email],
         )
diff --git a/app/grandchallenge/evaluation/models.py b/app/grandchallenge/evaluation/models.py
--- a/app/grandchallenge/evaluation/models.py
+++ b/app/grandchallenge/evaluation/models.py
@@ -38,6 +38,7 @@
 )
 from grandchallenge.evaluation.emails import (
     send_failed_evaluation_email,
+    send_missing_method_email,
     send_successful_evaluation_email,
 )
 from grandchallenge.evaluation.tasks import calculate_ranks
@@ -587,7 +588,7 @@ def create_evaluation(self):
         method = self.latest_ready_method
 
         if not method:
-            # TODO Email admins
+            send_missing_method_email(self)
             return
 
         evaluation = Evaluation.objects.create(submission=self, method=method)
@@ -608,9 +609,12 @@ def create_evaluation(self):
                     slug="predictions-csv-file"
                 )
             else:
-                raise NotImplementedError(
-                    f"Interface is not defined for {mimetype} files"
+                evaluation.update_status(
+                    status=Evaluation.FAILURE,
+                    stderr=f"{mimetype} files are not supported.",
+                    error_message=f"{mimetype} files are not supported.",
                 )
+                return
 
             evaluation.inputs.set(
                 [


Test patch:
diff --git a/app/tests/evaluation_tests/resources/submission.7z b/app/tests/evaluation_tests/resources/submission.7z
new file mode 100644
Binary files /dev/null and b/app/tests/evaluation_tests/resources/submission.7z differ
diff --git a/app/tests/evaluation_tests/test_tasks.py b/app/tests/evaluation_tests/test_tasks.py
--- a/app/tests/evaluation_tests/test_tasks.py
+++ b/app/tests/evaluation_tests/test_tasks.py
@@ -184,3 +184,34 @@ def test_set_evaluation_inputs(self):
         assert evaluation.status == evaluation.PENDING
         assert evaluation.error_message == ""
         assert evaluation.inputs.count() == 1
+
+
+@pytest.mark.django_db
+def test_non_zip_submission_failure(
+    client, evaluation_image, submission_file, settings
+):
+    # Override the celery settings
+    settings.task_eager_propagates = (True,)
+    settings.task_always_eager = (True,)
+
+    # Upload a submission and create an evaluation
+    eval_container, sha256 = evaluation_image
+    method = MethodFactory(
+        image__from_path=eval_container, image_sha256=sha256, ready=True
+    )
+
+    # Try with a 7z file
+    submission = SubmissionFactory(
+        predictions_file__from_path=Path(__file__).parent
+        / "resources"
+        / "submission.7z",
+        phase=method.phase,
+    )
+
+    # The evaluation method should return the correct answer
+    assert len(submission.evaluation_set.all()) == 1
+    evaluation = submission.evaluation_set.first()
+    assert evaluation.error_message.endswith(
+        "7z-compressed files are not supported."
+    )
+    assert evaluation.status == evaluation.FAILURE