Repository: AlexsLemonade/refinebio

Issue:
Processing info: Sample is not available but only tximport is run on it
Ex:
https://www.refine.bio/experiments/SRP074593/gene-expression-profiling-of-drosophila-s2r-cells-following-rnai-mediated-knockdown-of-transcription-factors

![Screen Shot 2020-01-03 at 2 58 39 PM](https://user-images.githubusercontent.com/15315514/71745903-8e155e00-2e39-11ea-92d4-de1927766356.png)


We should just show the submitter supplied info, since the samples aren't really processed through our pipeline



Discussion:
@kurtwheeler could this be one of the experiments where we ran tximport when not all samples were processed? This looks like a data integrity issue.

Resolving patch:
diff --git a/common/data_refinery_common/models/models.py b/common/data_refinery_common/models/models.py
--- a/common/data_refinery_common/models/models.py
+++ b/common/data_refinery_common/models/models.py
@@ -607,6 +607,25 @@ def remove_computed_files_from_s3(self):
         for computed_file in self.computedfile_set.all():
             computed_file.delete_s3_file()
 
+    def get_index_length(self):
+        """ Pull the index_length from one of the result annotations """
+        annotations = ComputationalResultAnnotation.objects.filter(result=self)
+
+        for annotation_json in annotations:
+            if "index_length" in annotation_json.data:
+                return annotation_json.data["index_length"]
+
+        return None
+
+    def get_quant_sf_file(self):
+        return (
+            ComputedFile.objects.filter(
+                result=self, filename="quant.sf", s3_key__isnull=False, s3_bucket__isnull=False,
+            )
+            .order_by("-id")
+            .first()
+        )
+
 
 class ComputationalResultAnnotation(models.Model):
     """ Non-standard information associated with an ComputationalResult """
diff --git a/common/data_refinery_common/rna_seq.py b/common/data_refinery_common/rna_seq.py
--- a/common/data_refinery_common/rna_seq.py
+++ b/common/data_refinery_common/rna_seq.py
@@ -75,7 +75,7 @@ def should_run_tximport(experiment: Experiment, results, is_tximport_job: bool):
 
 
 def get_quant_results_for_experiment(experiment: Experiment, filter_old_versions=True):
-    """Returns a list of salmon quant results from `experiment`."""
+    """Returns a queryset of salmon quant results from `experiment`."""
     # Subquery to calculate quant results
     # https://docs.djangoproject.com/en/2.2/ref/models/expressions/#subquery-expressions
 
@@ -119,28 +119,16 @@ def get_quant_files_for_results(results: List[ComputationalResult]):
     quant_files = []
 
     for result in results:
-        try:
-            quant_files.append(
-                ComputedFile.objects.filter(
-                    result=result,
-                    filename="quant.sf",
-                    s3_key__isnull=False,
-                    s3_bucket__isnull=False,
-                ).order_by("-id")[0]
-            )
-        except Exception as e:
-            try:
-                sample = result.samples.first()
-            except:
-                sample = None
-
+        quant_sf_file = result.get_quant_sf_file()
+        if quant_sf_file:
+            quant_files.append(quant_sf_file)
+        else:
             logger.exception(
                 "Salmon quant result found without quant.sf ComputedFile!",
                 quant_result=result.id,
-                sample=sample.id,
-                experiment=experiment.id,
+                sample=result.samples.first(),
             )
-            raise e
+            raise Exception("Salmon quant result found without quant.sf ComputedFile!")
 
     return quant_files
 
diff --git a/workers/data_refinery_workers/processors/salmon.py b/workers/data_refinery_workers/processors/salmon.py
--- a/workers/data_refinery_workers/processors/salmon.py
+++ b/workers/data_refinery_workers/processors/salmon.py
@@ -502,27 +502,23 @@ def _run_tximport_for_experiment(
     try:
         tximport_result = subprocess.run(cmd_tokens, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
     except Exception as e:
-        error_template = "Encountered error in R code while running tximport.R: {}"
-        error_message = error_template.format(str(e))
-        logger.error(error_message, processor_job=job_context["job_id"], experiment=experiment.id)
-        job_context["job"].failure_reason = error_message
-        job_context["success"] = False
-        return job_context
+        raise utils.ProcessorJobError(
+            "Encountered error in R code while running tximport.R: {}".format(str(e)),
+            success=False,
+            experiment=experiment.id,
+        )
 
     if tximport_result.returncode != 0:
-        error_template = "Found non-zero exit code from R code while running tximport.R: {}"
-        error_message = error_template.format(tximport_result.stderr.decode().strip())
-        logger.error(
-            error_message,
-            processor_job=job_context["job_id"],
+        raise utils.ProcessorJobError(
+            "Found non-zero exit code from R code while running tximport.R: {}".format(
+                tximport_result.stderr.decode().strip()
+            ),
+            success=False,
             experiment=experiment.id,
             quant_files=quant_files,
             cmd_tokens=cmd_tokens,
             quant_file_paths=quant_file_paths,
         )
-        job_context["job"].failure_reason = error_message
-        job_context["success"] = False
-        return job_context
 
     result.time_end = timezone.now()
     result.commands.append(" ".join(cmd_tokens))
@@ -531,7 +527,9 @@ def _run_tximport_for_experiment(
         processor_key = "TXIMPORT"
         result.processor = utils.find_processor(processor_key)
     except Exception as e:
-        return utils.handle_processor_exception(job_context, processor_key, e)
+        raise utils.ProcessorJobError(
+            "Failed to set processor: {}".format(e), success=False, processor_key=processor_key
+        )
 
     result.save()
     job_context["pipeline"].steps.append(result.id)
@@ -569,7 +567,8 @@ def _run_tximport_for_experiment(
         frame.to_csv(frame_path, sep="\t", encoding="utf-8")
 
         # The frame column header is based off of the path, which includes _output.
-        sample = Sample.objects.get(accession_code=frame.columns.values[0].replace("_output", ""))
+        sample_accession_code = frame.columns.values[0].replace("_output", "")
+        sample = Sample.objects.get(accession_code=sample_accession_code)
 
         computed_file = ComputedFile()
         computed_file.absolute_file_path = frame_path
@@ -614,46 +613,33 @@ def get_tximport_inputs(job_context: Dict) -> Dict:
     mapping to a list of paths to the quant.sf file for each sample in
     that experiment.
     """
-    experiments = Experiment.objects.filter(
-        samples=job_context["sample"]
-    )  # https://stackoverflow.com/a/18317340/763705
+    experiments = job_context["sample"].experiments.all()
 
     quantified_experiments = {}
     for experiment in experiments:
         # We only want to consider samples that we actually can run salmon on.
         eligible_samples = experiment.samples.filter(source_database="SRA", technology="RNA-SEQ")
-        num_eligible_samples = eligible_samples.count()
-        if num_eligible_samples == 0:
+        if not eligible_samples.exists():
             continue
 
         salmon_quant_results = get_quant_results_for_experiment(experiment)
         is_tximport_job = "is_tximport_only" in job_context and job_context["is_tximport_only"]
 
-        if is_tximport_job and salmon_quant_results.count() > 0:
+        first_salmon_quant_result = salmon_quant_results.first()
+        if is_tximport_job and first_salmon_quant_result:
             # If the job is only running tximport, then index_length
             # hasn't been set on the job context because we don't have
             # a raw file to run it on. Therefore pull it from one of
             # the result annotations.
-
-            annotations = ComputationalResultAnnotation.objects.filter(
-                result=salmon_quant_results[0]
-            )
-
-            for annotation_json in annotations:
-                if "index_length" in annotation_json.data:
-                    job_context["index_length"] = annotation_json.data["index_length"]
-                    break
-
-            if not "index_length" in job_context:
-                failure_reason = (
-                    "Found quant result without an annotation specifying its index length. "
-                    "Why did this happen?!?"
+            index_length = first_salmon_quant_result.get_index_length()
+            if index_length:
+                job_context["index_length"] = index_length
+            elif not "index_length" in job_context:
+                raise utils.ProcessorJobError(
+                    "Found quant result without an annotation specifying its index length. Why did this happen?!?",
+                    success=False,
+                    no_retry=True,
                 )
-                logger.error(failure_reason, processor_job=job_context["job_id"])
-                job_context["job"].failure_reason = failure_reason
-                job_context["job"].no_retry = True
-                job_context["success"] = False
-                return job_context
 
         if should_run_tximport(experiment, salmon_quant_results, is_tximport_job):
             quantified_experiments[experiment] = get_quant_files_for_results(salmon_quant_results)
@@ -668,13 +654,10 @@ def tximport(job_context: Dict) -> Dict:
     of genes_to_transcripts.txt.
     """
     tximport_inputs = job_context["tximport_inputs"]
+
     quantified_experiments = 0
     for experiment, quant_files in tximport_inputs.items():
         job_context = _run_tximport_for_experiment(job_context, experiment, quant_files)
-        # If `tximport` on any related experiment fails, exit immediately.
-        if not job_context["success"]:
-            return job_context
-
         quantified_experiments += 1
 
     if (
@@ -682,11 +665,9 @@ def tximport(job_context: Dict) -> Dict:
         and "is_tximport_job" in job_context
         and job_context["is_tximport_only"]
     ):
-        failure_reason = "Tximport job ran on no experiments... Why?!?!?"
-        logger.error(failure_reason, processor_job=job_context["job_id"])
-        job_context["job"].failure_reason = failure_reason
-        job_context["job"].no_retry = True
-        job_context["success"] = False
+        raise utils.ProcessorJobError(
+            "Tximport job ran on no experiments... Why?!?!?", success=False, no_retry=True
+        )
 
     return job_context
 


Test patch: