Repository: comic/grand-challenge.org

Issue:
Add algorithm jobs to the cleanup
An algorithm job was interrupted and is still in the started state. Check that the cleanup function is enabled for all `container_exec` jobs.


Discussion:

Resolving patch:
diff --git a/app/config/settings.py b/app/config/settings.py
--- a/app/config/settings.py
+++ b/app/config/settings.py
@@ -35,6 +35,7 @@ def strtobool(val) -> bool:
     re.compile(r"^/phpmyadmin.*"),
     re.compile(r"^/gen204.*"),
     re.compile(r"^/wp-content.*"),
+    re.compile(r"^/wp.*"),
     re.compile(r".*/trackback.*"),
     re.compile(r"^/site/.*"),
     re.compile(r"^/media/cache/.*"),
@@ -654,6 +655,26 @@ def strtobool(val) -> bool:
         "options": {"queue": "evaluation"},
         "schedule": timedelta(hours=1),
     },
+    "mark_long_running_algorithm_gpu_jobs_failed": {
+        "task": "grandchallenge.container_exec.tasks.mark_long_running_jobs_failed",
+        "kwargs": {
+            "app_label": "algorithms",
+            "model_name": "job",
+            "exclude": {"requires_gpu": False},
+        },
+        "options": {"queue": "gpu"},
+        "schedule": timedelta(hours=1),
+    },
+    "mark_long_running_algorithm_jobs_failed": {
+        "task": "grandchallenge.container_exec.tasks.mark_long_running_jobs_failed",
+        "kwargs": {
+            "app_label": "algorithms",
+            "model_name": "job",
+            "exclude": {"requires_gpu": True},
+        },
+        "options": {"queue": "evaluation"},
+        "schedule": timedelta(hours=1),
+    },
     "cache_retina_archive_data": {
         "task": "grandchallenge.retina_api.tasks.cache_archive_data",
         "schedule": timedelta(hours=1),
diff --git a/app/grandchallenge/container_exec/tasks.py b/app/grandchallenge/container_exec/tasks.py
--- a/app/grandchallenge/container_exec/tasks.py
+++ b/app/grandchallenge/container_exec/tasks.py
@@ -2,6 +2,7 @@
 import tarfile
 import uuid
 from datetime import timedelta
+from typing import Dict
 
 from celery import shared_task
 from django.apps import apps
@@ -177,7 +178,9 @@ def execute_job(
 
 
 @shared_task
-def mark_long_running_jobs_failed(*, app_label: str, model_name: str):
+def mark_long_running_jobs_failed(
+    *, app_label: str, model_name: str, exclude: Dict[str, str] = None
+):
     """
     Mark jobs that have been started but did not finish (maybe due to
     an unrecoverable hardware error). It will mark tasks FAILED that have the
@@ -199,6 +202,9 @@ def mark_long_running_jobs_failed(*, app_label: str, model_name: str):
         status=Job.STARTED,
     )
 
+    if exclude:
+        jobs_to_mark = jobs_to_mark.exclude(**exclude)
+
     for j in jobs_to_mark:
         j.update_status(status=Job.FAILURE, output="Evaluation timed out")
 


Test patch:
diff --git a/app/tests/container_exec_tests/test_tasks.py b/app/tests/container_exec_tests/test_tasks.py
--- a/app/tests/container_exec_tests/test_tasks.py
+++ b/app/tests/container_exec_tests/test_tasks.py
@@ -12,7 +12,7 @@
 @pytest.mark.django_db
 def test_mark_long_running_jobs_failed():
     # Started jobs should be unaffected
-    EvaluationJobFactory(status=EvaluationJob.STARTED)
+    j1 = EvaluationJobFactory(status=EvaluationJob.STARTED)
 
     # Long running jobs should be marked as failed
     j2 = EvaluationJobFactory(status=EvaluationJob.STARTED)
@@ -26,21 +26,29 @@ def test_mark_long_running_jobs_failed():
     j3.save()
 
     # Algorithm jobs should not be affected
-    AlgorithmJobFactory(status=AlgorithmJob.STARTED)
+    a = AlgorithmJobFactory(status=AlgorithmJob.STARTED)
 
     assert EvaluationJob.objects.all().count() == 3
     assert (
         AlgorithmJob.objects.filter(status=AlgorithmJob.STARTED).count() == 1
     )
+    assert (
+        EvaluationJob.objects.filter(status=EvaluationJob.FAILURE).count() == 0
+    )
+
+    assert j1.status == EvaluationJob.STARTED
+    assert j2.status == EvaluationJob.STARTED
+    assert j3.status == EvaluationJob.PENDING
+    assert a.status == AlgorithmJob.STARTED
 
     mark_long_running_jobs_failed(app_label="evaluation", model_name="job")
 
-    assert (
-        EvaluationJob.objects.filter(status=EvaluationJob.STARTED).count() == 1
-    )
-    assert (
-        EvaluationJob.objects.filter(status=EvaluationJob.PENDING).count() == 1
-    )
-    assert (
-        AlgorithmJob.objects.filter(status=AlgorithmJob.STARTED).count() == 1
-    )
+    j1.refresh_from_db()
+    j2.refresh_from_db()
+    j3.refresh_from_db()
+    a.refresh_from_db()
+
+    assert j1.status == EvaluationJob.STARTED
+    assert j2.status == EvaluationJob.FAILURE
+    assert j3.status == EvaluationJob.PENDING
+    assert a.status == AlgorithmJob.STARTED