Repository: AlexsLemonade/refinebio

Issue:
Set `nomad_job_id` for compendia jobs
### Context

I was trying to look for a compendia processor job where I only had the `nomad_job_id`

### Problem or idea

Looks like we don't set the `nomad_job_id` for the compendia processor jobs

```sql
$ select * from processor_jobs where pipeline_applied='COMPENDIA' and start_time > now() - '48 hours'::interval;
    id    | pipeline_applied | no_retry | ram_amount | volume_index |          start_time           |           end_time            | success | nomad_job_id | num_retries | retried |      worker_id      | worker_version  |                                                         failure_reason                                                          |          created_at         
  |         last_modified         | retried_job_id 
----------+------------------+----------+------------+--------------+-------------------------------+-------------------------------+---------+--------------+-------------+---------+---------------------+-----------------+---------------------------------------------------------------------------------------------------------------------------------+-----------------------------
--+-------------------------------+----------------
  7723013 | COMPENDIA        | f        |       2048 |              | 2019-09-26 19:06:39.774994+00 | 2019-09-27 00:19:36.860523+00 | f       |              |           0 | f       | i-0aad2ec47c070fad1 | v1.27.14-hotfix | Failed to upload computed file.                                                                                                 | 2019-09-26 19:06:39.708193+0
0 | 2019-09-27 00:19:36.860547+00 |               
  7540834 | COMPENDIA        | f        |       2048 |              | 2019-09-26 17:21:04.618606+00 |                               |         |              |           0 | f       | i-0aad2ec47c070fad1 | v1.27.14-hotfix |                                                                                                                                 | 2019-09-26 17:21:03.267152+0
0 | 2019-09-26 17:21:04.618685+00 |               
  7540448 | COMPENDIA        | f        |       2048 |              | 2019-09-26 17:20:57.091504+00 |                               |         |              |           0 | f       | i-0aad2ec47c070fad1 | v1.27.14-hotfix |                                                                                                                                 | 2019-09-26 17:20:55.186121+0
0 | 2019-09-26 17:20:57.091538+00 |               
```

### Solution or next step

The `nomad_job_id` should have a value, similar to other jobs that we have.



Discussion:

Resolving patch:
diff --git a/common/data_refinery_common/utils.py b/common/data_refinery_common/utils.py
--- a/common/data_refinery_common/utils.py
+++ b/common/data_refinery_common/utils.py
@@ -441,7 +441,7 @@ def _aggregate_nomad_jobs(aggregated_jobs):
 
     return nomad_pending_jobs, nomad_running_jobs
 
-def queryset_iterator(queryset, page_size):
+def queryset_iterator(queryset, page_size = 2000):
     """ use the performant paginator to iterate over a queryset """
     paginator = PerformantPaginator(queryset, page_size)
     page = paginator.page()
diff --git a/workers/data_refinery_workers/processors/management/commands/create_compendia.py b/workers/data_refinery_workers/processors/management/commands/create_compendia.py
--- a/workers/data_refinery_workers/processors/management/commands/create_compendia.py
+++ b/workers/data_refinery_workers/processors/management/commands/create_compendia.py
@@ -1,28 +1,17 @@
 import sys
 
 from django.core.management.base import BaseCommand
+
 from data_refinery_common.job_lookup import ProcessorPipeline
 from data_refinery_common.logging import get_and_configure_logger
 from data_refinery_common.message_queue import send_job
-from data_refinery_common.performant_pagination.pagination import PerformantPaginator
-
-from data_refinery_common.models import (
-    Experiment,
-    Sample,
-    Organism,
-    ProcessorJob,
-    Dataset,
-    ProcessorJobDatasetAssociation,
-    ExperimentOrganismAssociation,
-    ExperimentSampleAssociation
-)
-from data_refinery_workers.processors import create_compendia
+from data_refinery_common.models import (Dataset, Experiment, Organism,
+                                         ProcessorJob,
+                                         ProcessorJobDatasetAssociation)
+from data_refinery_common.utils import queryset_iterator
 
 logger = get_and_configure_logger(__name__)
 
-PAGE_SIZE = 2000
-
-
 def create_job_for_organism(organism=Organism, quant_sf_only=False, svd_algorithm='ARPACK'):
     """Returns a compendia job for the provided organism.
 
@@ -30,16 +19,9 @@ def create_job_for_organism(organism=Organism, quant_sf_only=False, svd_algorith
     """
     data = {}
     experiments = Experiment.objects.filter(organisms=organism).prefetch_related('samples')
-    paginator = PerformantPaginator(experiments, PAGE_SIZE)
-    page = paginator.page()
-    while True:
-        for experiment in page.object_list:
-            data[experiment.accession_code] = list(experiment.samples.filter(organism=organism).values_list('accession_code', flat=True))
-
-        if not page.has_next():
-            break
-        else:
-            page = paginator.page(page.next_page_number())
+
+    for experiment in queryset_iterator(experiments):
+        data[experiment.accession_code] = list(experiment.samples.filter(organism=organism).values_list('accession_code', flat=True))
 
     job = ProcessorJob()
     job.pipeline_applied = ProcessorPipeline.CREATE_COMPENDIA.value
@@ -105,16 +87,12 @@ def handle(self, *args, **options):
         if options["svd_algorithm"] in ['ARPACK', 'RANDOMIZED', 'NONE']:
             svd_algorithm = options["svd_algorithm"]
 
-        logger.error(all_organisms)
+        logger.debug(all_organisms)
 
-        if all_organisms.count() > 1:
-            for organism in all_organisms:
-                logger.error(organism)
-                job = create_job_for_organism(organism, quant_sf_only, svd_algorithm)
-                logger.info("Sending CREATE_COMPENDIA for Organism", job_id=str(job.pk), organism=str(organism))
-                send_job(ProcessorPipeline.CREATE_COMPENDIA, job)
-        else:
-            job = create_job_for_organism(all_organisms[0], quant_sf_only, svd_algorithm)
-            create_compendia.create_compendia(job.id)
+        for organism in all_organisms:
+            logger.debug(organism)
+            job = create_job_for_organism(organism, quant_sf_only, svd_algorithm)
+            logger.info("Sending CREATE_COMPENDIA for Organism", job_id=str(job.pk), organism=str(organism))
+            send_job(ProcessorPipeline.CREATE_COMPENDIA, job)
 
         sys.exit(0)


Test patch: