Repository: comic/grand-challenge.org

Issue:
Challenge runtime statistics are not correct
Looking at the Tiger statistics, for Segmentation and Detection (Final) phase:

Average algorithm job duration for Segmentation and Detection (Final):	25 minutes
Total algorithm job duration for Segmentation and Detection (Final):	4 days

There has been 1 submission to this phase:

![MicrosoftTeams-image (3)](https://user-images.githubusercontent.com/12661555/172356728-3c3656ca-cbf9-48e6-a886-29435e11e660.png)

Looking at a couple of those results they seem to take ~30 seconds each, and maybe 50 jobs. I think what is happening is that the algorithm was used for multiple phases, and there are different versions, which are inflating the runtime statistics.




Discussion:
Hm tricky. We could additionally filter on the viewer_groups when retrieving the jobs to calculate the average duration for. That would limit the jobs to jobs for that challenge, but it could still include jobs from another phase of the same challenge, so not really a solution to the problem. But I don't think we have any way of distinguishing jobs any further than this, or do you see another way?
Maybe only include jobs that have input civs from the archive that is linked to the phase?

Resolving patch:
diff --git a/app/grandchallenge/pages/views.py b/app/grandchallenge/pages/views.py
--- a/app/grandchallenge/pages/views.py
+++ b/app/grandchallenge/pages/views.py
@@ -17,7 +17,6 @@
 
 from grandchallenge.algorithms.models import Job
 from grandchallenge.core.mixins import UserIsStaffMixin
-from grandchallenge.evaluation.models import Submission
 from grandchallenge.evaluation.utils import SubmissionKindChoices
 from grandchallenge.pages.forms import PageCreateForm, PageUpdateForm
 from grandchallenge.pages.models import Page
@@ -152,14 +151,9 @@ def delete(self, request, *args, **kwargs):
 
 
 def get_average_job_duration_for_phase(phase):
-    algorithm_images = Submission.objects.filter(
-        phase__pk=phase.pk
-    ).values_list("algorithm_image__pk")
     jobs = Job.objects.filter(
-        algorithm_image__pk__in=algorithm_images,
-        status=Job.SUCCESS,
-        creator=None,
-    )
+        outputs__evaluation_evaluations_as_input__submission__phase=phase,
+    ).distinct()
     duration_dict = {
         "average_duration": jobs.average_duration(),
         "total_duration": jobs.total_duration(),
@@ -172,14 +166,9 @@ class ChallengeStatistics(LoginRequiredMixin, UserIsStaffMixin, TemplateView):
 
     def get_context_data(self, **kwargs):
         context = super().get_context_data()
-        phases = (
-            self.request.challenge.phase_set.filter(
-                submission_kind=SubmissionKindChoices.ALGORITHM
-            )
-            .select_related("archive")
-            .prefetch_related("archive__items__values")
-            .all()
-        )
+        phases = self.request.challenge.phase_set.filter(
+            submission_kind=SubmissionKindChoices.ALGORITHM
+        ).all()
         duration_dict = {}
         for phase in phases:
             duration_dict[phase.title] = get_average_job_duration_for_phase(


Test patch:
diff --git a/app/tests/pages_tests/test_pages.py b/app/tests/pages_tests/test_pages.py
--- a/app/tests/pages_tests/test_pages.py
+++ b/app/tests/pages_tests/test_pages.py
@@ -1,9 +1,22 @@
+from datetime import timedelta
 from itertools import chain
 
 import pytest
 from django.db.models import BLANK_CHOICE_DASH
+from django.utils.timezone import now
 
+from grandchallenge.components.models import (
+    ComponentInterface,
+    ComponentInterfaceValue,
+)
+from grandchallenge.evaluation.utils import SubmissionKindChoices
 from grandchallenge.pages.models import Page
+from grandchallenge.pages.views import get_average_job_duration_for_phase
+from tests.algorithms_tests.factories import (
+    AlgorithmImageFactory,
+    AlgorithmJobFactory,
+)
+from tests.evaluation_tests.factories import EvaluationFactory, PhaseFactory
 from tests.factories import ChallengeFactory, PageFactory, UserFactory
 from tests.utils import get_view_for_user, validate_admin_only_view
 
@@ -337,3 +350,37 @@ def test_challenge_statistics_page_permissions(client):
         challenge=challenge,
     )
     response.status_code = 200
+
+
+@pytest.mark.django_db
+def test_average_job_duration_calculation():
+    challenge = ChallengeFactory()
+    phase1, phase2 = PhaseFactory.create_batch(
+        2, challenge=challenge, submission_kind=SubmissionKindChoices.ALGORITHM
+    )
+
+    ai = AlgorithmImageFactory(ready=True)
+
+    j1 = AlgorithmJobFactory(
+        algorithm_image=ai,
+        started_at=now() - timedelta(minutes=1),
+        completed_at=now(),
+    )
+    _ = AlgorithmJobFactory(
+        algorithm_image=ai,
+        started_at=now() - timedelta(minutes=2),
+        completed_at=now(),
+    )
+    j1.outputs.add(
+        ComponentInterfaceValue.objects.create(
+            interface=ComponentInterface.objects.get(slug="metrics-json-file"),
+            value=True,
+        )
+    )
+    e1 = EvaluationFactory(submission__phase=phase1)
+    e1.inputs.add(j1.outputs.first())
+
+    duration = get_average_job_duration_for_phase(phase=phase1)
+
+    assert duration["average_duration"].seconds == timedelta(minutes=1).seconds
+    assert duration["total_duration"].seconds == timedelta(minutes=1).seconds