Repository: comic/grand-challenge.org

Issue:
Evaluations incorrectly marked as Failed
When running an algorithm submission sometimes the evaluation is incorrectly marked as failed as not all of the jobs have been scheduled.

Also if an algorithm job fails other jobs will still be scheduled, there needs to be a guard in `create_algorithm_jobs_for_evaluation`.


Discussion:

Resolving patch:
diff --git a/app/grandchallenge/components/backends/utils.py b/app/grandchallenge/components/backends/utils.py
--- a/app/grandchallenge/components/backends/utils.py
+++ b/app/grandchallenge/components/backends/utils.py
@@ -61,13 +61,6 @@ def parse_structured_log(*, log: str) -> Optional[ParsedLog]:
     if structured_log["internal"] is False:
         # Defensive, in case the type of structured_log["internal"] is str
         return ParsedLog(message=message, source=source)
-    else:
-        if source == SourceChoices.STDOUT:
-            logger.info(f"Internal log: {message}")
-        elif source == SourceChoices.STDERR:
-            logger.warning(f"Internal log: {message}")
-        else:
-            raise ValueError("Invalid log structure")
 
 
 def safe_extract(*, src: File, dest: Path):
diff --git a/app/grandchallenge/evaluation/tasks.py b/app/grandchallenge/evaluation/tasks.py
--- a/app/grandchallenge/evaluation/tasks.py
+++ b/app/grandchallenge/evaluation/tasks.py
@@ -178,6 +178,15 @@ def retry_with_delay():
     )
     evaluation = Evaluation.objects.get(pk=evaluation_pk)
 
+    if evaluation.status not in {
+        evaluation.PENDING,
+        evaluation.EXECUTING_PREREQUISITES,
+    }:
+        logger.info(
+            f"Nothing to do: evaluation is {evaluation.get_status_display()}."
+        )
+        return
+
     # Only the challenge admins should be able to view these jobs, never
     # the algorithm editors as these are participants - they must never
     # be able to see the test data.
@@ -228,20 +237,28 @@ def retry_with_delay():
                 max_jobs=max_jobs,
                 time_limit=evaluation.submission.phase.algorithm_time_limit,
             )
-
-            if not jobs:
-                # No more jobs created from this task, so everything must be ready
-                # for evaluation
-                set_evaluation_inputs.signature(
-                    kwargs={"evaluation_pk": str(evaluation.pk)},
-                    immutable=True,
-                ).apply_async()
-
     except (TooManyJobsScheduled, LockError) as error:
         logger.info(f"Retrying task due to: {error}")
         retry_with_delay()
         raise
 
+    if not jobs:
+        if max_jobs is None:
+            # No more jobs created from this task, so everything must be
+            # ready for evaluation, handles archives with only one item
+            set_evaluation_inputs.signature(
+                kwargs={"evaluation_pk": str(evaluation.pk)},
+                immutable=True,
+            ).apply_async()
+        else:
+            evaluation.update_status(
+                status=Evaluation.FAILURE,
+                error_message=(
+                    "No jobs could be scheduled for this evaluation, "
+                    "do the archive items have the correct interfaces?"
+                ),
+            )
+
 
 @shared_task(**settings.CELERY_TASK_DECORATOR_KWARGS["acks-late-micro-short"])
 @transaction.atomic
@@ -294,11 +311,11 @@ def set_evaluation_inputs(*, evaluation_pk):
     )
 
     has_pending_jobs = (
-        Job.objects.filter(
+        Job.objects.active()
+        .filter(
             algorithm_image__submission__evaluation=evaluation_pk,
             creator__isnull=True,  # Evaluation inference jobs have no creator
         )
-        .exclude(status__in=[Job.SUCCESS, Job.FAILURE, Job.CANCELLED])
         .exists()
     )
 
@@ -375,8 +392,6 @@ def set_evaluation_inputs(*, evaluation_pk):
             evaluation.save()
 
             on_commit(evaluation.execute)
-        else:
-            handle_failed_jobs(evaluation_pk=evaluation_pk)
 
 
 def filter_by_creators_most_recent(*, evaluations):


Test patch:
diff --git a/app/tests/evaluation_tests/test_tasks.py b/app/tests/evaluation_tests/test_tasks.py
--- a/app/tests/evaluation_tests/test_tasks.py
+++ b/app/tests/evaluation_tests/test_tasks.py
@@ -235,19 +235,6 @@ def setUp(self):
         self.jobs = jobs
         self.output_civs = output_civs
 
-    def test_unsuccessful_jobs_fail_evaluation(self):
-        self.jobs[0].status = Job.FAILURE
-        self.jobs[0].save()
-
-        set_evaluation_inputs(evaluation_pk=self.evaluation.pk)
-
-        self.evaluation.refresh_from_db()
-        assert self.evaluation.status == self.evaluation.FAILURE
-        assert (
-            self.evaluation.error_message
-            == "The algorithm failed on one or more cases."
-        )
-
     def test_set_evaluation_inputs(self):
         set_evaluation_inputs(evaluation_pk=self.evaluation.pk)