Repository: AlexsLemonade/refinebio

Issue:
Quantpendia doesn't seem to run
### Context

Just triggered the quantpendia job for Danio

### Problem or idea

It doesn't seem to have started:

```
ubuntu@ip-10-0-80-121:~$ ./run_management_command.sh create_compendia --organisms=DANIO_RERIO --quant-sf-only=True --svd-algorithm=None
fd6293be1671e7c5ebf63a96473ddac6d1d084a58dfecb85262487f31198aec3
ubuntu@ip-10-0-80-121:~$ docker logs fd6293be1671e7c5ebf63a96473ddac6d1d084a58dfecb85262487f31198aec3
2019-10-21 16:53:29,668 i-09b7a30710d2781fa [volume: -1] data_refinery_foreman.foreman.management.commands.create_compendia INFO [job_id: 29291464] [organism: DANIO_RERIO]: Sending CREATE_COMPENDIA for Organism
```

That job doesn't have a `start_date` on the db.

```
    id    |  pipeline_applied  | no_retry | ram_amount | volume_index | start_time | end_time | success |                 nomad_job_id                  | num_retries | retried | worker_id | worker_version | failure_reason |          created_at           |         last_modified         | retried_job_id 
----------+--------------------+----------+------------+--------------+------------+----------+---------+-----------------------------------------------+-------------+---------+-----------+----------------+----------------+-------------------------------+-------------------------------+----------------
 29291464 | CREATE_QUANTPENDIA | f        |       2048 |              |            |          |         | CREATE_COMPENDIA/dispatch-1571676809-09e79eb6 |           0 | f       |           |                |                | 2019-10-21 16:53:29.607529+00 | 2019-10-21 16:53:29.686031+00 |               
(1 row)
```

And seems to be an error on nomad when creating the job:

```
ubuntu@ip-10-0-242-7:~$ nomad status CREATE_COMPENDIA/dispatch-1571676809-09e79eb6
ID            = CREATE_COMPENDIA/dispatch-1571676809-09e79eb6
Name          = CREATE_COMPENDIA/dispatch-1571676809-09e79eb6
Submit Date   = 2019-10-21T16:53:29Z
Type          = batch
Priority      = 80
Datacenters   = dc1
Status        = pending
Periodic      = false
Parameterized = false

Summary
Task Group  Queued  Starting  Running  Failed  Complete  Lost
jobs        1       0         0        0       0         0

Placement Failure
Task Group "jobs":
  * Constraint "${meta.is_smasher} = true" filtered 1 nodes
  * Resources exhausted on 1 nodes
  * Dimension "memory" exhausted on 1 nodes

Allocations
No allocations placed
```


### Solution or next step

We need to investigate what happened.



Discussion:

Resolving patch:
diff --git a/common/data_refinery_common/message_queue.py b/common/data_refinery_common/message_queue.py
--- a/common/data_refinery_common/message_queue.py
+++ b/common/data_refinery_common/message_queue.py
@@ -56,6 +56,8 @@ def send_job(job_type: Enum, job, is_dispatch=False) -> bool:
         nomad_job = ProcessorPipeline.QN_REFERENCE.value
     elif job_type is ProcessorPipeline.CREATE_COMPENDIA:
         nomad_job = ProcessorPipeline.CREATE_COMPENDIA.value
+    elif job_type is ProcessorPipeline.CREATE_QUANTPENDIA:
+        nomad_job = ProcessorPipeline.CREATE_QUANTPENDIA.value
     elif job_type is ProcessorPipeline.AGILENT_TWOCOLOR_TO_PCL:
         # Agilent twocolor uses the same job specification as Affy.
         nomad_job = ProcessorPipeline.AFFY_TO_PCL.value
diff --git a/foreman/data_refinery_foreman/foreman/management/commands/create_compendia.py b/foreman/data_refinery_foreman/foreman/management/commands/create_compendia.py
--- a/foreman/data_refinery_foreman/foreman/management/commands/create_compendia.py
+++ b/foreman/data_refinery_foreman/foreman/management/commands/create_compendia.py
@@ -91,9 +91,12 @@ def handle(self, *args, **options):
 
         logger.debug('Generating compendia for organisms', organisms=all_organisms)
 
+        job_pipeline = ProcessorPipeline.CREATE_QUANTPENDIA if quant_sf_only else ProcessorPipeline.CREATE_COMPENDIA
+
         for organism in all_organisms:
             job = create_job_for_organism(organism, quant_sf_only, svd_algorithm)
-            logger.info("Sending CREATE_COMPENDIA for Organism", job_id=str(job.pk), organism=str(organism))
-            send_job(ProcessorPipeline.CREATE_COMPENDIA, job)
-
-        sys.exit(0)
+            logger.info("Sending compendia job for Organism",
+                        job_id=str(job.pk),
+                        organism=str(organism),
+                        quant_sf_only=quant_sf_only)
+            send_job(job_pipeline, job)
diff --git a/workers/data_refinery_workers/processors/create_quantpendia.py b/workers/data_refinery_workers/processors/create_quantpendia.py
--- a/workers/data_refinery_workers/processors/create_quantpendia.py
+++ b/workers/data_refinery_workers/processors/create_quantpendia.py
@@ -71,7 +71,6 @@ def create_result_objects(job_context: Dict) -> Dict:
     shutil.copy("/home/user/README_QUANT.md", job_context["output_dir"] + "/README.md")
 
     archive_path = shutil.make_archive(final_zip_base, 'zip', job_context["output_dir"])
-
     compendia_organism = _get_organisms(job_context['samples']).first()
     compendia_version = _get_next_compendia_version(compendia_organism)
 


Test patch:
diff --git a/foreman/data_refinery_foreman/foreman/management/commands/test_create_compendia.py b/foreman/data_refinery_foreman/foreman/management/commands/test_create_compendia.py
--- a/foreman/data_refinery_foreman/foreman/management/commands/test_create_compendia.py
+++ b/foreman/data_refinery_foreman/foreman/management/commands/test_create_compendia.py
@@ -1,5 +1,6 @@
 from django.test import TransactionTestCase
 from django.utils import timezone
+from django.core.management import call_command
 
 from data_refinery_common.job_lookup import ProcessorPipeline
 from data_refinery_common.models import (ComputationalResult,
@@ -12,7 +13,8 @@
                                          ProcessorJobDatasetAssociation,
                                          Sample,
                                          SampleComputedFileAssociation,
-                                         SampleResultAssociation)
+                                         SampleResultAssociation,
+                                         ExperimentOrganismAssociation)
 from data_refinery_foreman.surveyor.test_end_to_end import wait_for_job
 
 from .create_compendia import create_job_for_organism
@@ -22,30 +24,38 @@ class CompendiaCommandTestCase(TransactionTestCase):
     def test_quantpendia_command(self):
         self.make_test_data()
 
-        homo_sapiens = Organism.get_object_for_name("HOMO_SAPIENS")
+        args = []
+        options = {'organisms': 'HOMO_SAPIENS', 'quant_sf_only': True}
+        call_command('create_compendia', *args, **options)
 
         start_time = timezone.now()
-        compendia_job = create_job_for_organism(homo_sapiens, True)
-        wait_for_job(compendia_job, ProcessorJob, start_time)
+        processor_job = ProcessorJob.objects\
+            .filter(pipeline_applied='CREATE_QUANTPENDIA')\
+            .order_by('-created_at')\
+            .first()
+        processor_job = wait_for_job(processor_job, ProcessorJob, start_time)
+        self.assertTrue(processor_job.success)
 
         # assert we have a quantpendia for human
+        homo_sapiens = Organism.get_object_for_name("HOMO_SAPIENS")
         quantpendias = ComputedFile.objects.filter(is_compendia=True, quant_sf_only=True, compendia_organism=homo_sapiens).first()
         self.assertTrue(quantpendias)
 
     def make_test_data(self):
-        job = ProcessorJob()
-        job.pipeline_applied = ProcessorPipeline.CREATE_QUANTPENDIA.value
-        job.save()
+        homo_sapiens = Organism.get_object_for_name("HOMO_SAPIENS")
 
         experiment = Experiment()
         experiment.accession_code = "GSE51088"
         experiment.save()
 
+        xoa = ExperimentOrganismAssociation()
+        xoa.experiment=experiment
+        xoa.organism=homo_sapiens
+        xoa.save()
+
         result = ComputationalResult()
         result.save()
 
-        homo_sapiens = Organism.get_object_for_name("HOMO_SAPIENS")
-
         sample = Sample()
         sample.accession_code = 'GSM1237818'
         sample.title = 'GSM1237818'
@@ -84,16 +94,3 @@ def make_test_data(self):
         assoc.sample = sample
         assoc.computed_file = computed_file
         assoc.save()
-
-        ds = Dataset()
-        ds.data = {'GSE51088': ['GSM1237818']}
-        ds.aggregate_by = 'EXPERIMENT'
-        ds.scale_by = 'STANDARD'
-        ds.email_address = "null@derp.com"
-        ds.quant_sf_only = True # Make the dataset include quant.sf files only
-        ds.save()
-
-        pjda = ProcessorJobDatasetAssociation()
-        pjda.processor_job = job
-        pjda.dataset = ds
-        pjda.save()