Repository: AlexsLemonade/refinebio

Issue:
IndexError: list index out of range
`job["ID"]` is `'TXIMPORT'` that's why this fails

---

Sentry Issue: [STAGING-REFINEBIO-API-F4](https://sentry.io/organizations/greenelab/issues/928698207/?project=1236826&referrer=github_integration)

```
IndexError: list index out of range
(6 additional frame(s) were not displayed)
...
  File "rest_framework/views.py", line 443, in handle_exception
    self.raise_uncaught_exception(exc)
  File "rest_framework/views.py", line 480, in dispatch
    response = handler(request, *args, **kwargs)
  File "data_refinery_api/views.py", line 1055, in get
    nomad_stats = self._get_nomad_jobs_breakdown()
  File "data_refinery_api/views.py", line 1216, in _get_nomad_jobs_breakdown
    nomad_pending_jobs_by_volume, nomad_running_jobs_by_volume = self._aggregate_nomad_jobs_by_volume(parameterized_jobs)
  File "data_refinery_api/views.py", line 1156, in _aggregate_nomad_jobs_by_volume
    volume_id = "_".join(job["ID"].split("_")[-2])

IndexError: list index out of range
```


Discussion:

Resolving patch:
diff --git a/api/data_refinery_api/views.py b/api/data_refinery_api/views.py
--- a/api/data_refinery_api/views.py
+++ b/api/data_refinery_api/views.py
@@ -3,7 +3,8 @@
 import mailchimp3
 import nomad
 from typing import Dict
-
+from itertools import groupby
+from re import match
 from django.conf import settings
 from django.db.models import Count, Prefetch, DateTimeField
 from django.db.models.functions import Trunc
@@ -1051,17 +1052,13 @@ def get(self, request, format=None):
             data['input_data_size'] = self._get_input_data_size()
             data['output_data_size'] = self._get_output_data_size()
 
-        try:
-            nomad_stats = self._get_nomad_jobs_breakdown()
-            data['nomad_running_jobs'] = nomad_stats["nomad_running_jobs"]
-            data['nomad_pending_jobs'] = nomad_stats["nomad_pending_jobs"]
-            data['nomad_running_jobs_by_type'] = nomad_stats["nomad_running_jobs_by_type"]
-            data['nomad_pending_jobs_by_type'] = nomad_stats["nomad_pending_jobs_by_type"]
-            data['nomad_running_jobs_by_volume'] = nomad_stats["nomad_running_jobs_by_volume"]
-            data['nomad_pending_jobs_by_volume'] = nomad_stats["nomad_pending_jobs_by_volume"]
-        except nomad.api.exceptions.BaseNomadException:
-            # Nomad is not available right now, so exclude these.
-            pass
+        nomad_stats = self._get_nomad_jobs_breakdown()
+        data['nomad_running_jobs'] = nomad_stats["nomad_running_jobs"]
+        data['nomad_pending_jobs'] = nomad_stats["nomad_pending_jobs"]
+        data['nomad_running_jobs_by_type'] = nomad_stats["nomad_running_jobs_by_type"]
+        data['nomad_pending_jobs_by_type'] = nomad_stats["nomad_pending_jobs_by_type"]
+        data['nomad_running_jobs_by_volume'] = nomad_stats["nomad_running_jobs_by_volume"]
+        data['nomad_pending_jobs_by_volume'] = nomad_stats["nomad_pending_jobs_by_volume"]
 
         return Response(data)
 
@@ -1095,112 +1092,61 @@ def _samples_processed_last_hour(self):
         start = current_date - timedelta(hours=1)
         return Sample.processed_objects.filter(created_at__range=(start, current_date)).count()
 
-    def _aggregate_nomad_jobs_by_type(self, jobs: Dict):
-        """Aggregates the pending and running job counts for each Nomad job type.
+    def _aggregate_nomad_jobs(self, aggregated_jobs):
+        """Aggregates the job counts.
 
         This is accomplished by using the stats that each
         parameterized job has about its children jobs.
 
         `jobs` should be a response from the Nomad API's jobs endpoint.
         """
-        job_types = set()
-        for job in jobs:
-            # Surveyor jobs don't have ids and RAM, so handle them specially.
-            if job["ID"].startswith("SURVEYOR"):
-                job_types.add("SURVEYOR")
-            elif job["ID"] == "SMASHER" or job["ID"] == "DOWNLOADER":
-                job_types.add(job["ID"])
-            else:
-                # Strips out the last two underscores like so:
-                # SALMON_1_16384 -> SALMON
-                job_type = "_".join(job["ID"].split("_")[0:-2])
-                job_types.add(job_type)
-
-        nomad_running_jobs_by_type = {}
-        nomad_pending_jobs_by_type = {}
-        for job_type in job_types:
-            # This will count SURVEYOR_DISPATCHER jobs as SURVEYOR
-            # jobs, but I think that's fine since we barely ever run
-            # SURVEYOR_DISPATCHER jobs and won't need to monitor them
-            # through the dashboard.
-            same_jobs = [job for job in jobs if job["ID"].startswith(job_type)]
-
+        nomad_running_jobs = {}
+        nomad_pending_jobs = {}
+        for (aggregate_key, group) in aggregated_jobs:
+            if not aggregate_key: continue
             aggregated_pending = 0
             aggregated_running = 0
-            for job in same_jobs:
+            for job in group:
                 children = job["JobSummary"]["Children"]
                 aggregated_pending = aggregated_pending + children["Pending"]
                 aggregated_running = aggregated_running + children["Running"]
 
-            nomad_pending_jobs_by_type[job_type] = aggregated_pending
-            nomad_running_jobs_by_type[job_type] = aggregated_running
-
-        return nomad_pending_jobs_by_type, nomad_running_jobs_by_type
+            nomad_pending_jobs[aggregate_key] = aggregated_pending
+            nomad_running_jobs[aggregate_key] = aggregated_running
 
-    def _aggregate_nomad_jobs_by_volume(self, jobs: Dict):
-        """Aggregates the job counts for each EBS volume.
+        return nomad_pending_jobs, nomad_running_jobs
 
-        This is accomplished by using the stats that each
-        parameterized job has about its children jobs.
-
-        `jobs` should be a response from the Nomad API's jobs endpoint.
-        """
-        volume_ids = set()
-        for job in jobs:
-            # These job types don't have volume indices, so we just won't count them.
-            if not job["ID"].startswith("SURVEYOR") \
-               and job["ID"] != "SMASHER" \
-               and job["ID"] != "DOWNLOADER":
-                # Strips out the volume ID like so:
-                # SALMON_1_16384 -> 1
-                volume_id = "_".join(job["ID"].split("_")[-2])
-                volume_ids.add(volume_id)
-
-        nomad_running_jobs_by_volume = {}
-        nomad_pending_jobs_by_volume = {}
-        for volume_id in volume_ids:
-            if job["ID"].startswith("SURVEYOR") \
-               or job["ID"] == "SMASHER" \
-               or job["ID"] == "DOWNLOADER":
-                continue
+    def _get_job_details(self, job):
+        """Given a Nomad Job, as returned by the API, returns the type and volume id that should be used
+        when aggregating for the stats endpoint"""
+        # Surveyor jobs don't have ids and RAM, so handle them specially.
+        if job["ID"].startswith("SURVEYOR"):
+            return "SURVEYOR", False
 
-            def job_has_same_volume(job: Dict) -> bool:
-                """Returns true if the job is on the same volume as this iteration of the loop.
+        # example SALMON_1_2323
+        name_match = match(r"(?P<type>\w+)_(?P<volume_id>\d+)_\d+$", job["ID"])
+        if not name_match: return False, False
+        
+        return name_match.group('type'), name_match.group('volume_id')
 
-                These job types don't have volume indices, so we just
-                won't count them. We theoretically could try, but it
-                really would be more trouble than it's worth and this
-                endpoint is already going to have a hard time returning
-                a response in time.
-                """
-                return not job["ID"].startswith("SURVEYOR") \
-                    and job["ID"] != "SMASHER" \
-                    and job["ID"] != "DOWNLOADER" \
-                    and job["ID"].split("_")[-2] == volume_id
-
-            jobs_with_same_volume = [job for job in jobs if job_has_same_volume(job)]
-
-            aggregated_pending = 0
-            aggregated_running = 0
-            for job in jobs_with_same_volume:
-                children = job["JobSummary"]["Children"]
-                aggregated_pending = aggregated_pending + children["Pending"]
-                aggregated_running = aggregated_running + children["Running"]
-
-            nomad_pending_jobs_by_volume["volume_" + str(volume_id)] = aggregated_pending
-            nomad_running_jobs_by_volume["volume_" + str(volume_id)] = aggregated_running
+    def _get_nomad_jobs(self):
+        """Calls nomad service and return all jobs"""
+        try:
+            nomad_host = get_env_variable("NOMAD_HOST")
+            nomad_port = get_env_variable("NOMAD_PORT", "4646")
+            nomad_client = nomad.Nomad(nomad_host, port=int(nomad_port), timeout=30)
+            return nomad_client.jobs.get_jobs()
 
-        return nomad_pending_jobs_by_volume, nomad_running_jobs_by_volume
+        except nomad.api.exceptions.BaseNomadException:
+            # Nomad is not available right now
+            return []
 
     def _get_nomad_jobs_breakdown(self):
-        nomad_host = get_env_variable("NOMAD_HOST")
-        nomad_port = get_env_variable("NOMAD_PORT", "4646")
-        nomad_client = nomad.Nomad(nomad_host, port=int(nomad_port), timeout=30)
-
-        jobs = nomad_client.jobs.get_jobs()
+        jobs = self._get_nomad_jobs()
         parameterized_jobs = [job for job in jobs if job['ParameterizedJob']]
 
-        nomad_pending_jobs_by_type, nomad_running_jobs_by_type = self._aggregate_nomad_jobs_by_type(parameterized_jobs)
+        aggregated_jobs_by_type = groupby(parameterized_jobs, lambda job: self._get_job_details(job)[0])
+        nomad_pending_jobs_by_type, nomad_running_jobs_by_type = self._aggregate_nomad_jobs(aggregated_jobs_by_type)
 
         # To get the total jobs for running and pending, the easiest
         # AND the most efficient way is to sum up the stats we've
@@ -1213,7 +1159,8 @@ def _get_nomad_jobs_breakdown(self):
         for job_type, num_jobs in nomad_pending_jobs_by_type.items():
             nomad_pending_jobs = nomad_pending_jobs + num_jobs
 
-        nomad_pending_jobs_by_volume, nomad_running_jobs_by_volume = self._aggregate_nomad_jobs_by_volume(parameterized_jobs)
+        aggregated_jobs_by_volume = groupby(parameterized_jobs, lambda job: self._get_job_details(job)[1])
+        nomad_pending_jobs_by_volume, nomad_running_jobs_by_volume = self._aggregate_nomad_jobs(aggregated_jobs_by_volume)
 
         return {
             "nomad_pending_jobs": nomad_pending_jobs,


Test patch:
diff --git a/api/data_refinery_api/tests.py b/api/data_refinery_api/tests.py
--- a/api/data_refinery_api/tests.py
+++ b/api/data_refinery_api/tests.py
@@ -25,7 +25,7 @@
     ProcessorSerializer,
     SurveyJobSerializer,
 )
-from data_refinery_api.views import ExperimentList
+from data_refinery_api.views import ExperimentList, Stats
 from data_refinery_common.utils import get_env_variable
 from data_refinery_common.models import (
     ComputationalResult,
@@ -909,6 +909,113 @@ def raise_error(_):
         self.assertEqual(response.status_code, 500)
         mock_client.captureMessage.assert_called()
 
+class StatsTestCases(APITestCase):
+    @patch.object(Stats, '_get_nomad_jobs')
+    def test_nomad_stats_empty(self, mock_get_nomad_jobs):
+        mock_get_nomad_jobs.return_value = []
+        response = self.client.get(reverse('stats'))
+        self.assertEqual(response.status_code, 200)
+        self.assertEqual(response.json()['nomad_running_jobs'], 0)
+        self.assertEqual(response.json()['nomad_pending_jobs'], 0)
+
+    @patch.object(Stats, '_get_nomad_jobs')
+    def test_nomad_stats(self, mock_get_nomad_jobs):
+        mock_get_nomad_jobs.return_value = StatsTestCases.MOCK_NOMAD_RESPONSE
+        response = self.client.get(reverse('stats'))
+        self.assertEqual(response.status_code, 200)
+        self.assertEqual(response.json()['nomad_running_jobs'], 2)
+        self.assertEqual(response.json()['nomad_pending_jobs'], 0)
+        self.assertEqual(response.json()['nomad_running_jobs_by_type']['SALMON'], 2)
+        self.assertEqual(response.json()['nomad_running_jobs_by_volume']['1'], 1)
+        self.assertEqual(response.json()['nomad_running_jobs_by_volume']['2'], 1)
+
+    MOCK_NOMAD_RESPONSE = [
+        {
+            'CreateIndex': 5145, 
+            'ID': 'TXIMPORT', 
+            'JobModifyIndex': 5145, 
+            'JobSummary': {
+            'Children': {
+                'Dead': 0, 
+                'Pending': 0, 
+                'Running': 0
+            }, 
+            'CreateIndex': 5145,
+            'JobID': 'TXIMPORT',
+            'ModifyIndex': 5145,
+            'Namespace': 'default',
+            'Summary': {}
+            }, 
+            'ModifyIndex': 5145, 
+            'Name': 'TXIMPORT', 
+            'ParameterizedJob': True, 
+            'ParentID': '', 
+            'Periodic': False, 
+            'Priority': 50, 
+            'Status': 'running', 
+            'StatusDescription': '', 
+            'Stop': False, 
+            'SubmitTime': 1552322030836469355, 
+            'Type': 'batch'
+        },
+        {
+            'CreateIndex': 5145, 
+            'ID': 'SALMON_1_2323', 
+            'JobModifyIndex': 5145, 
+            'JobSummary': {
+            'Children': {
+                'Dead': 0, 
+                'Pending': 0, 
+                'Running': 1
+            }, 
+            'CreateIndex': 5145, 
+            'JobID': 'SALMON_1_2323', 
+            'ModifyIndex': 5145, 
+            'Namespace': 'default', 
+            'Summary': {}
+            }, 
+            'ModifyIndex': 5145, 
+            'Name': 'SALMON_1_2323', 
+            'ParameterizedJob': True, 
+            'ParentID': '', 
+            'Periodic': False, 
+            'Priority': 50, 
+            'Status': 'running', 
+            'StatusDescription': '', 
+            'Stop': False, 
+            'SubmitTime': 1552322030836469355, 
+            'Type': 'batch'
+        },
+        {
+            'CreateIndex': 5145, 
+            'ID': 'SALMON_2_2323', 
+            'JobModifyIndex': 5145, 
+            'JobSummary': {
+            'Children': {
+                'Dead': 0, 
+                'Pending': 0, 
+                'Running': 1
+            }, 
+            'CreateIndex': 5145, 
+            'JobID': 'SALMON_1_2323', 
+            'ModifyIndex': 5145, 
+            'Namespace': 'default', 
+            'Summary': {}
+            }, 
+            'ModifyIndex': 5145, 
+            'Name': 'SALMON_1_2323', 
+            'ParameterizedJob': True, 
+            'ParentID': '', 
+            'Periodic': False, 
+            'Priority': 50, 
+            'Status': 'running', 
+            'StatusDescription': '', 
+            'Stop': False, 
+            'SubmitTime': 1552322030836469355, 
+            'Type': 'batch'
+        }
+    ]
+
 class ESTestCases(APITestCase):
 
     def test_es_endpoint(self):