Repository: AlexsLemonade/refinebio

Issue:
We should be able to detect more than one p-value column in our Illumina pipeline
### Context

#2275 

### Problem or idea

It looks like right now we only detect one p-value column for each Illumina sample (under certain conditions), which causes some samples to fail processing.

### Solution or next step

Modify our p-value detection code to handle multiple p-value columns.



Discussion:

Resolving patch:
diff --git a/foreman/data_refinery_foreman/foreman/main.py b/foreman/data_refinery_foreman/foreman/main.py
--- a/foreman/data_refinery_foreman/foreman/main.py
+++ b/foreman/data_refinery_foreman/foreman/main.py
@@ -742,6 +742,14 @@ def requeue_processor_job(last_job: ProcessorJob) -> None:
                 new_ram_amount = 8192
             elif new_ram_amount == 8192:
                 new_ram_amount = 32768
+        elif (
+            last_job.pipeline_applied == "ILLUMINA_TO_PCL"
+            and "non-zero exit status -9" in last_job.failure_reason
+        ):
+            if new_ram_amount == 2048:
+                new_ram_amount = 4096
+            elif new_ram_amount == 4096:
+                new_ram_amount = 8192
 
     volume_index = last_job.volume_index
     # Make sure volume_index is set to something, unless it's a
diff --git a/foreman/data_refinery_foreman/surveyor/geo.py b/foreman/data_refinery_foreman/surveyor/geo.py
--- a/foreman/data_refinery_foreman/surveyor/geo.py
+++ b/foreman/data_refinery_foreman/surveyor/geo.py
@@ -328,9 +328,12 @@ def create_experiment_and_samples_from_api(
                 sample_object.save()
                 logger.debug("Created Sample: " + str(sample_object))
 
+                metadata = sample.metadata
+                metadata["geo_columns"] = list(sample.columns.index)
+
                 sample_annotation = SampleAnnotation()
                 sample_annotation.sample = sample_object
-                sample_annotation.data = sample.metadata
+                sample_annotation.data = metadata
                 sample_annotation.is_ccdl = False
                 sample_annotation.save()
 
diff --git a/workers/data_refinery_workers/processors/illumina.py b/workers/data_refinery_workers/processors/illumina.py
--- a/workers/data_refinery_workers/processors/illumina.py
+++ b/workers/data_refinery_workers/processors/illumina.py
@@ -134,12 +134,12 @@ def _detect_columns(job_context: Dict) -> Dict:
         else:
             job_context["probeId"] = headers[predicted_header]
 
-        # Then the detection Pvalue string, which is always(?) some form of 'Detection Pval'
+        # Then check to make sure a detection pvalue exists, which is always(?) some form of
+        # 'Detection Pval'
         for header in headers:
             # check if header contains something like "detection pval"
-            pvalue_header = re.match(r"(detection)(\W?)(pval\w*)", header, re.IGNORECASE)
+            pvalue_header = re.search(r"(detection)(\W?)(pval\w*)", header, re.IGNORECASE)
             if pvalue_header:
-                job_context["detectionPval"] = pvalue_header.string
                 break
         else:
             job_context["job"].failure_reason = "Could not detect PValue column!"
@@ -150,12 +150,12 @@ def _detect_columns(job_context: Dict) -> Dict:
         # Then, finally, create an absolutely bonkers regular expression
         # which will explicitly hit on any sample which contains a sample
         # ID _and_ ignores the magical word 'BEAD', etc. Great!
-        column_ids = ""
+        column_ids = set()
         for sample in job_context["samples"]:
             for offset, header in enumerate(headers, start=1):
 
                 if sample.title == header:
-                    column_ids = column_ids + str(offset) + ","
+                    column_ids.add(offset)
                     continue
 
                 # Sometimes the title might actually be in the description field.
@@ -166,7 +166,7 @@ def _detect_columns(job_context: Dict) -> Dict:
                 for annotation in sample.sampleannotation_set.filter(is_ccdl=False):
                     try:
                         if annotation.data.get("description", "")[0] == header:
-                            column_ids = column_ids + str(offset) + ","
+                            column_ids.add(offset)
                             continue_me = True
                             break
                     except Exception:
@@ -178,7 +178,7 @@ def _detect_columns(job_context: Dict) -> Dict:
                     continue
 
                 if header.upper().replace(" ", "_") == "RAW_VALUE":
-                    column_ids = column_ids + str(offset) + ","
+                    columns_ids.add(offset)
                     continue
 
                 if (
@@ -188,17 +188,15 @@ def _detect_columns(job_context: Dict) -> Dict:
                     and "ARRAY_STDEV" not in header.upper()
                     and "PVAL" not in header.upper().replace(" ", "").replace("_", "")
                 ):
-                    column_ids = column_ids + str(offset) + ","
+                    column_ids.add(offset)
                     continue
 
         for offset, header in enumerate(headers, start=1):
             if "AVG_Signal" in header:
-                column_ids = column_ids + str(offset) + ","
+                column_ids.add(offset)
                 continue
 
-        # Remove the trailing comma
-        column_ids = column_ids[:-1]
-        job_context["columnIds"] = column_ids
+        job_context["columnIds"] = ",".join(map(lambda id: str(id), column_ids))
     except Exception as e:
         job_context["job"].failure_reason = str(e)
         job_context["success"] = False
@@ -325,8 +323,6 @@ def _run_illumina(job_context: Dict) -> Dict:
             job_context["probeId"],
             "--expression",
             job_context["columnIds"],
-            "--detection",
-            job_context["detectionPval"],
             "--platform",
             job_context["platform"],
             "--inputFile",
@@ -356,6 +352,38 @@ def _run_illumina(job_context: Dict) -> Dict:
     return job_context
 
 
+def _get_sample_for_column(column: str, job_context: Dict) -> Sample:
+    # First of all check if the title is the column name
+    try:
+        return job_context["samples"].get(title=column)
+    except Sample.DoesNotExist:
+        pass
+
+    # If the column name is not the title, maybe they used the convention
+    # <SAMPLE_TITLE>(.AVG)?_Signal
+    title_match = re.match(r"(?P<title>.*?)(\.AVG)?_Signal", column)
+    if title_match is not None:
+        try:
+            return job_context["samples"].get(title=title_match.group("title"))
+        except Sample.DoesNotExist:
+            pass
+
+    # Or maybe they also have a named detection pvalue column using the same
+    # naming scheme
+    name_match = re.match(r"(?P<name>.*)\.AVG_Signal", column)
+    if name_match is not None:
+        try:
+            return job_context["samples"].get(
+                sampleannotation__data__geo_columns__contains="{}.Detection Pval".format(
+                    name_match.group("name")
+                )
+            )
+        except Sample.DoesNotExist:
+            pass
+
+    return None
+
+
 def _create_result_objects(job_context: Dict) -> Dict:
     """ Create the ComputationalResult objects after a Scan run is complete """
 
@@ -387,16 +415,17 @@ def _create_result_objects(job_context: Dict) -> Dict:
         frame.to_csv(frame_path, sep="\t", encoding="utf-8")
 
         # This needs to be the same as the ones in the job context!
-        try:
-            sample = job_context["samples"].get(title=frame.columns.values[0])
-        except Sample.DoesNotExist:
-            logger.error(
-                "Could not find sample for column while splitting Illumina file.",
-                title=frame.columns.values[0],
-                processor_job=job_context["job_id"],
-                file_path=big_tsv,
+        sample = _get_sample_for_column(frame.columns.values[0], job_context)
+        if sample is None:
+            job_context["job"].failure_reason = (
+                "Could not find sample for column "
+                + frame.columns.values[0]
+                + " while splitting Illumina file "
+                + big_tsv
             )
-            continue
+            job_context["success"] = False
+            job_context["job"].no_retry = True
+            return job_context
 
         computed_file = ComputedFile()
         computed_file.absolute_file_path = frame_path


Test patch:
diff --git a/foreman/data_refinery_foreman/surveyor/test_end_to_end.py b/foreman/data_refinery_foreman/surveyor/test_end_to_end.py
--- a/foreman/data_refinery_foreman/surveyor/test_end_to_end.py
+++ b/foreman/data_refinery_foreman/surveyor/test_end_to_end.py
@@ -847,3 +847,45 @@ def test_unmated_reads(self):
 
             # The downloader job will take a while to complete. Let's not wait.
             print(downloader_job.kill_nomad_job())
+
+
+# This test uses the special tag "manual" because it should only be run from the "test_survey.sh" script
+class SurveyTestCase(TransactionTestCase):
+    @tag("manual")
+    def test_survey(self):
+        """Survey the given sample"""
+
+        # Clear out pre-existing work dirs so there's no conflicts:
+        self.env = EnvironmentVarGuard()
+        self.env.set("RUNING_IN_CLOUD", "False")
+        with self.env:
+            for work_dir in glob.glob(LOCAL_ROOT_DIR + "/processor_job_*"):
+                shutil.rmtree(work_dir)
+
+            survey_job = surveyor.survey_experiment(
+                get_env_variable("ACCESSION"), get_env_variable("SURVEYOR")
+            )
+
+            self.assertTrue(survey_job.success)
+
+            downloader_jobs = DownloaderJob.objects.all()
+            self.assertGreater(downloader_jobs.count(), 0)
+
+            logger.info("Survey Job finished, waiting for Downloader Jobs to complete.")
+            start_time = timezone.now()
+            for downloader_job in downloader_jobs:
+                downloader_job = wait_for_job(downloader_job, DownloaderJob, start_time)
+                self.assertTrue(downloader_job.success)
+
+            processor_jobs = ProcessorJob.objects.all().exclude(
+                abort=True
+            )  # exclude aborted processor jobs
+            self.assertGreater(processor_jobs.count(), 0)
+
+            logger.info("Downloader Jobs finished, waiting for processor Jobs to complete.")
+            start_time = timezone.now()
+            for processor_job in processor_jobs:
+                processor_job = wait_for_job(processor_job, ProcessorJob, start_time)
+                if not processor_job.success:
+                    logger.error(processor_job.failure_reason)
+                self.assertTrue(processor_job.success)
diff --git a/foreman/run_tests.sh b/foreman/run_tests.sh
--- a/foreman/run_tests.sh
+++ b/foreman/run_tests.sh
@@ -55,4 +55,4 @@ docker run \
        --add-host=elasticsearch:"$ES_HOST_IP" \
        --env-file foreman/environments/test \
        --volume "$volume_directory":/home/user/data_store \
-       -it ccdlstaging/dr_foreman bash -c "$(run_tests_with_coverage "$@")"
+       -it ccdlstaging/dr_foreman bash -c "$(run_tests_with_coverage --exclude-tag=manual "$@")"
diff --git a/foreman/test_survey.sh b/foreman/test_survey.sh
new file mode 100755
--- /dev/null
+++ b/foreman/test_survey.sh
@@ -0,0 +1,104 @@
+#!/bin/sh
+
+# Script for testing the surveying of an accession manually (e.g. from a dataset request)
+print_options() {
+    cat << EOF
+There are two required arguments for this script that come from survey_experiment():
+-s specifies which surveyor to use.
+-a specifies which experiment accession should be surveyed.
+EOF
+}
+
+while getopts ":s:a:h" opt; do
+    case $opt in
+    s)
+        export SURVEYOR=$OPTARG
+        ;;
+    a)
+        export ACCESSION=$OPTARG
+        ;;
+    h)
+        print_description
+        echo
+        print_options
+        exit 0
+        ;;
+    \?)
+        echo "Invalid option: -$OPTARG" >&2
+        print_options >&2
+        exit 1
+        ;;
+    :)
+        echo "Option -$OPTARG requires an argument." >&2
+        print_options >&2
+        exit 1
+        ;;
+    esac
+done
+
+if [ -z "$SURVEYOR" ]; then
+    echo 'Error: must specify which surveyor to use with -s' >&2
+    exit 1
+fi
+
+if [ -z "$ACCESSION" ]; then
+    echo 'Error: must specify which accession to survey with -a' >&2
+    exit 1
+fi
+
+# This script should always run as if it were being called from
+# the directory it lives in.
+script_directory="$(perl -e 'use File::Basename;
+ use Cwd "abs_path";
+ print dirname(abs_path(@ARGV[0]));' -- "$0")"
+cd "$script_directory" || exit
+
+# Set up the data volume directory if it does not already exist.
+# Since the end-to-end tests are run from the Foreman image, use the
+# top level test_volume rather than one nested within the foreman
+# directory.
+project_root=$(cd .. && pwd)
+volume_directory="$project_root/test_volume"
+if [ ! -d "$volume_directory" ]; then
+    mkdir "$volume_directory"
+    chmod -R a+rwX "$volume_directory"
+fi
+
+# However in order to give Docker access to all the code we have to
+# move up a level
+cd ..
+
+# Ensure that Nomad is running first
+# The -f option in `pgrep` means that pgrep checks the full command line
+if ! pgrep -f test_nomad > /dev/null; then
+    echo "You must start the nomad test environment first with:" >&2
+    echo "sudo -E ./scripts/run_nomad.sh -e test" >&2
+    exit 1
+# Then ensure postgres is running
+elif ! [ "$(docker ps --filter name=drdb -q)" ]; then
+    echo "You must start Postgres first with:" >&2
+    echo "./scripts/run_postgres.sh" >&2
+    exit 1
+# Then ensure elasticsearch is running
+elif ! [ "$(docker ps --filter name=dres -q)" ]; then
+    echo "You must start Elasticsearch first with:" >&2
+    echo "./scripts/run_es.sh" >&2
+    exit 1
+fi
+
+./scripts/prepare_image.sh -i foreman -s foreman
+
+. ./scripts/common.sh
+DB_HOST_IP=$(get_docker_db_ip_address)
+ES_HOST_IP=$(get_docker_es_ip_address)
+HOST_IP=$(get_ip_address || echo 127.0.0.1)
+
+docker run \
+       --add-host=database:"$DB_HOST_IP" \
+       --add-host=nomad:"$HOST_IP" \
+       --add-host=elasticsearch:"$ES_HOST_IP" \
+       --env-file foreman/environments/test \
+       --volume "$volume_directory":/home/user/data_store \
+       -e SURVEYOR="$SURVEYOR" \
+       -e ACCESSION="$ACCESSION" \
+       -it ccdlstaging/dr_foreman bash -c "python3 manage.py test --tag=manual ."
diff --git a/workers/data_refinery_workers/processors/test_geo.py b/workers/data_refinery_workers/processors/test_geo.py
--- a/workers/data_refinery_workers/processors/test_geo.py
+++ b/workers/data_refinery_workers/processors/test_geo.py
@@ -109,6 +109,7 @@ def test_illumina_to_pcl(self):
 
         job = prepare_illumina_job(organism)
         final_context = illumina.illumina_to_pcl(job.pk)
+        self.assertTrue(final_context["success"])
 
         for sample in final_context["samples"]:
             smashme = sample.get_most_recent_smashable_result_file()
@@ -161,7 +162,7 @@ def test_good_detection(self):
 
         sample = Sample()
         sample.accession_code = "ABCD-1234"
-        sample.title = "hypoxia_Signal"
+        sample.title = "CB CD34+ hypoxia"
         sample.organism = organism
         sample.save()
 
@@ -170,7 +171,19 @@ def test_good_detection(self):
         sample_assoc.sample = sample
         sample_assoc.save()
 
+        sample2 = Sample()
+        sample2.accession_code = "ABCD-1235"
+        sample2.title = "CB CD34+ normoxia"
+        sample2.organism = organism
+        sample2.save()
+
+        sample_assoc2 = OriginalFileSampleAssociation()
+        sample_assoc2.original_file = og_file
+        sample_assoc2.sample = sample2
+        sample_assoc2.save()
+
         final_context = illumina.illumina_to_pcl(pj.pk)
+        self.assertTrue(final_context["success"])
         self.assertEqual(final_context["platform"], "illuminaHumanv3")
 
         for key in final_context["samples"][0].sampleannotation_set.all()[0].data.keys():