Repository: AlexsLemonade/refinebio

Issue:
Quantpendia command timmed out for MUS_MUSCULUS
Error:

```
ubuntu@ip-10-0-127-71:~$ docker logs inspiring_goldberg
2019-11-11 22:18:45,252 i-04bb28b499152d24f [volume: -1] data_refinery_foreman.foreman.management.commands.create_quantpendia INFO [organism: HOMO_SAPIENS] [job_id: 29380081]: Sending compendia job for Organism
Traceback (most recent call last):
  File "/usr/local/lib/python3.5/dist-packages/django/db/backends/utils.py", line 85, in _execute
    return self.cursor.execute(sql, params)
psycopg2.extensions.QueryCanceledError: canceling statement due to statement timeout


The above exception was the direct cause of the following exception:

Traceback (most recent call last):
  File "manage.py", line 23, in <module>
    execute_from_command_line(sys.argv)
  File "/usr/local/lib/python3.5/dist-packages/django/core/management/__init__.py", line 381, in execute_from_command_line
    utility.execute()
  File "/usr/local/lib/python3.5/dist-packages/django/core/management/__init__.py", line 375, in execute
    self.fetch_command(subcommand).run_from_argv(self.argv)
  File "/usr/local/lib/python3.5/dist-packages/django/core/management/base.py", line 316, in run_from_argv
    self.execute(*args, **cmd_options)
  File "/usr/local/lib/python3.5/dist-packages/django/core/management/base.py", line 353, in execute
    output = self.handle(*args, **options)
  File "/home/user/data_refinery_foreman/foreman/management/commands/create_quantpendia.py", line 83, in handle
    job = create_job_for_organism(organism)
  File "/home/user/data_refinery_foreman/foreman/management/commands/create_quantpendia.py", line 34, in create_job_for_organism
    data[experiment.accession_code] = list(samples_with_quantsf)
  File "/usr/local/lib/python3.5/dist-packages/django/db/models/query.py", line 268, in __iter__
    self._fetch_all()
  File "/usr/local/lib/python3.5/dist-packages/django/db/models/query.py", line 1186, in _fetch_all
    self._result_cache = list(self._iterable_class(self))
  File "/usr/local/lib/python3.5/dist-packages/django/db/models/query.py", line 176, in __iter__
    for row in compiler.results_iter(chunked_fetch=self.chunked_fetch, chunk_size=self.chunk_size):
  File "/usr/local/lib/python3.5/dist-packages/django/db/models/sql/compiler.py", line 1017, in results_iter
    results = self.execute_sql(MULTI, chunked_fetch=chunked_fetch, chunk_size=chunk_size)
  File "/usr/local/lib/python3.5/dist-packages/django/db/models/sql/compiler.py", line 1065, in execute_sql
    cursor.execute(sql, params)
  File "/usr/local/lib/python3.5/dist-packages/raven/contrib/django/client.py", line 127, in execute
    return real_execute(self, sql, params)
  File "/usr/local/lib/python3.5/dist-packages/django/db/backends/utils.py", line 68, in execute
    return self._execute_with_wrappers(sql, params, many=False, executor=self._execute)
  File "/usr/local/lib/python3.5/dist-packages/django/db/backends/utils.py", line 77, in _execute_with_wrappers
    return executor(sql, params, many, context)
  File "/usr/local/lib/python3.5/dist-packages/django/db/backends/utils.py", line 85, in _execute
    return self.cursor.execute(sql, params)
  File "/usr/local/lib/python3.5/dist-packages/django/db/utils.py", line 89, in __exit__
    raise dj_exc_value.with_traceback(traceback) from exc_value
  File "/usr/local/lib/python3.5/dist-packages/django/db/backends/utils.py", line 85, in _execute
    return self.cursor.execute(sql, params)
django.db.utils.OperationalError: canceling statement due to statement timeout
```

Worked for human.


Discussion:

Resolving patch:
diff --git a/foreman/data_refinery_foreman/foreman/management/commands/create_quantpendia.py b/foreman/data_refinery_foreman/foreman/management/commands/create_quantpendia.py
--- a/foreman/data_refinery_foreman/foreman/management/commands/create_quantpendia.py
+++ b/foreman/data_refinery_foreman/foreman/management/commands/create_quantpendia.py
@@ -1,4 +1,5 @@
 import sys
+import time
 
 from django.core.management.base import BaseCommand
 
@@ -8,37 +9,19 @@
 from data_refinery_common.models import (Dataset, Experiment, Organism,
                                          ProcessorJob,
                                          ProcessorJobDatasetAssociation)
-from data_refinery_common.utils import queryset_iterator
+from data_refinery_common.utils import queryset_page_iterator
 
 logger = get_and_configure_logger(__name__)
 
 
 def create_job_for_organism(organism: Organism):
     """Returns a quantpendia job for the provided organism."""
-    data = {}
-    experiments = Experiment.objects.filter(
-            organisms=organism,
-            samples__results__computedfile__filename='quant.sf'
-        )\
-        .distinct()
-
-    for experiment in queryset_iterator(experiments):
-        # only include the samples from the target organism that have quant.sf files
-        samples_with_quantsf = experiment.samples\
-            .filter(
-                organism=organism,
-                results__computedfile__filename='quant.sf'
-            )\
-            .values_list('accession_code', flat=True)\
-            .distinct()
-        data[experiment.accession_code] = list(samples_with_quantsf)
-
     job = ProcessorJob()
     job.pipeline_applied = ProcessorPipeline.CREATE_QUANTPENDIA.value
     job.save()
 
     dset = Dataset()
-    dset.data = data
+    dset.data = build_dataset(organism)
     dset.scale_by = 'NONE'
     dset.aggregate_by = 'EXPERIMENT'
     dset.quantile_normalize = False
@@ -53,6 +36,42 @@ def create_job_for_organism(organism: Organism):
 
     return job
 
+def build_dataset(organism: Organism):
+    data = {}
+    experiments = Experiment.objects.filter(
+            organisms=organism,
+            technology='RNA-SEQ',
+        )\
+        .distinct()
+
+    for experiment_page in queryset_page_iterator(experiments):
+        for experiment in experiment_page:
+            # only include the samples from the target organism that have quant.sf files
+            experiment_samples = experiment.samples\
+                .filter(organism=organism, technology='RNA-SEQ')
+            # split the query into two so to avoid timeouts.
+            # assume processed rna-seq samples have a quant.sf file
+            processed_samples_with_quantsf = experiment_samples\
+                .filter(is_processed=True)\
+                .values_list('accession_code', flat=True)
+            # and only check for quant file for unprocessed samples
+            unprocessed_samples_with_quantsf = experiment_samples\
+                .filter(
+                    is_processed=False,
+                    results__computedfile__filename='quant.sf'
+                )\
+                .values_list('accession_code', flat=True)\
+                .distinct()
+
+            sample_accession_codes = list(processed_samples_with_quantsf) \
+                + list(unprocessed_samples_with_quantsf)
+
+            if (sample_accession_codes):
+                data[experiment.accession_code] = sample_accession_codes
+
+        time.sleep(5)
+
+    return data
 
 class Command(BaseCommand):
 
@@ -75,7 +94,7 @@ def handle(self, *args, **options):
             # only generate the quantpendia for organisms that have some samples
             # with quant.sf files.
             has_quantsf_files = organism.sample_set\
-                .filter(results__computedfile__filename='quant.sf')\
+                .filter(technology='RNA-SEQ', results__computedfile__filename='quant.sf')\
                 .exists()
             if not has_quantsf_files:
                 continue


Test patch:
diff --git a/foreman/data_refinery_foreman/foreman/management/commands/test_create_quantpendia.py b/foreman/data_refinery_foreman/foreman/management/commands/test_create_quantpendia.py
--- a/foreman/data_refinery_foreman/foreman/management/commands/test_create_quantpendia.py
+++ b/foreman/data_refinery_foreman/foreman/management/commands/test_create_quantpendia.py
@@ -52,6 +52,7 @@ def get_organism_with_qn_target():
 def make_test_data(organism):
     experiment = Experiment()
     experiment.accession_code = "GSE51088"
+    experiment.technology='RNA-SEQ'
     experiment.save()
 
     xoa = ExperimentOrganismAssociation()
@@ -66,6 +67,7 @@ def make_test_data(organism):
     sample.accession_code = 'GSM1237818'
     sample.title = 'GSM1237818'
     sample.organism = organism
+    sample.technology = 'RNA-SEQ'
     sample.save()
 
     sra = SampleResultAssociation()
diff --git a/workers/data_refinery_workers/processors/test_create_quantpendia.py b/workers/data_refinery_workers/processors/test_create_quantpendia.py
--- a/workers/data_refinery_workers/processors/test_create_quantpendia.py
+++ b/workers/data_refinery_workers/processors/test_create_quantpendia.py
@@ -38,6 +38,7 @@ def test_create_quantpendia(self):
         sample.accession_code = 'GSM1237818'
         sample.title = 'GSM1237818'
         sample.organism = homo_sapiens
+        sample.technology = 'RNA-SEQ'
         sample.save()
 
         sra = SampleResultAssociation()