Repository: AlexsLemonade/refinebio

Issue:
Create command to delete invalid compendias
### Context

https://github.com/AlexsLemonade/refinebio/pull/1734

Compendia need a QN target so they can be quantile normalized. We need 100 samples on the sample platform to create a QN target. Therefore any organisms without 100 samples on the same platform should be ineligible for a compendium.

### Problem or idea

We created a compendium for Bos Inidicus which you can see only has 57 samples: https://www.refine.bio/search?filter_order=organism&organism=bos_indicus

### Solution or next step

In https://github.com/AlexsLemonade/refinebio/pull/1734 I should have prevented us from trying to create a compendium for this situation.

However, we should still:
  * Look into how the compendia was successful despite not having a QN target to do that step.
  * Delete this and any other compendia that should have failed to be created in the first place.



Discussion:

Resolving patch:
diff --git a/common/data_refinery_common/models/models.py b/common/data_refinery_common/models/models.py
--- a/common/data_refinery_common/models/models.py
+++ b/common/data_refinery_common/models/models.py
@@ -560,6 +560,11 @@ def save(self, *args, **kwargs):
         self.last_modified = current_time
         return super(ComputationalResult, self).save(*args, **kwargs)
 
+    def remove_computed_files_from_s3(self):
+        """ Removes all associated computed files from S3. Use this before deleting a computational result. """
+        for computed_file in self.computedfile_set.all():
+                computed_file.delete_s3_file()
+
 
 class ComputationalResultAnnotation(models.Model):
     """ Non-standard information associated with an ComputationalResult """
@@ -1128,13 +1133,17 @@ def delete_s3_file(self, force=False):
 
         try:
             S3.delete_object(Bucket=self.s3_bucket, Key=self.s3_key)
-            return True
         except:
             logger.exception("Failed to delete S3 object for Computed File.",
                              computed_file=self.id,
                              s3_object=self.s3_key)
             return False
 
+        self.s3_key = None
+        self.s3_bucket = None
+        self.save()
+        return True
+
     def get_synced_file_path(self, force=False, path=None):
         """ Fetches the absolute file path to this ComputedFile, fetching from S3 if it
         isn't already available locally. """
diff --git a/workers/data_refinery_workers/processors/management/commands/create_qn_target.py b/workers/data_refinery_workers/processors/management/commands/create_qn_target.py
--- a/workers/data_refinery_workers/processors/management/commands/create_qn_target.py
+++ b/workers/data_refinery_workers/processors/management/commands/create_qn_target.py
@@ -72,20 +72,19 @@ def handle(self, *args, **options):
                 organisms = Organism.objects.all()
 
             for organism in organisms:
-                samples = Sample.processed_objects.filter(organism=organism, has_raw=True, technology="MICROARRAY", is_processed=True)
+                if not organism_can_have_qn_target(organism):
+                    logger.error("Organism does not have any platform with enough samples to generate a qn target",
+                                 organism=organism,
+                                 min=options["min"])
+                    continue
+
+                samples = organism.sample_set.filter(has_raw=True, technology="MICROARRAY", is_processed=True)
                 if samples.count() == 0:
                     logger.error("No processed samples for organism.",
                         organism=organism,
                         count=samples.count()
                         )
                     continue
-                if samples.count() < options['min']:
-                    logger.error("Proccessed samples don't meet minimum threshhold",
-                        organism=organism,
-                        count=samples.count(),
-                        min=options["min"]
-                    )
-                    continue
 
                 if options["platform"] is None:
                     platform_counts = samples.values('platform_accession_code').annotate(dcount=Count('platform_accession_code')).order_by('-dcount')
@@ -127,3 +126,16 @@ def handle(self, *args, **options):
                     print(":(")
         else:
             qn_reference.create_qn_reference(options["job_id"])
+
+
+def organism_can_have_qn_target(organism: Organism, sample_threshold=100):
+    """ Check that the organism has more than `sample_threshold` samples on
+    some microarray platform """
+    microarray_platforms = organism.sample_set\
+        .filter(has_raw=True, technology="MICROARRAY", is_processed=True)\
+        .values('platform_accession_code')\
+        .annotate(count=Count('id'))\
+        .filter(count__gt=sample_threshold)
+
+    return microarray_platforms.exists()
+
diff --git a/workers/data_refinery_workers/processors/management/commands/remove_invalid_computational_results.py b/workers/data_refinery_workers/processors/management/commands/remove_invalid_computational_results.py
new file mode 100644
--- /dev/null
+++ b/workers/data_refinery_workers/processors/management/commands/remove_invalid_computational_results.py
@@ -0,0 +1,92 @@
+
+from django.core.management.base import BaseCommand
+
+from data_refinery_common.logging import get_and_configure_logger
+from data_refinery_common.models import (ComputationalResult,
+                                         ComputationalResultAnnotation,
+                                         ComputedFile, Organism)
+
+from .create_qn_target import organism_can_have_qn_target
+
+logger = get_and_configure_logger(__name__)
+
+
+class Command(BaseCommand):
+    def add_arguments(self, parser):
+        parser.add_argument('--dry-run',
+                            default=False,
+                            action='store_true',
+                            help='Prints resulting actions without actually running them.')
+        parser.add_argument("--min",
+                            type=int,
+                            default=100,
+                            help=("Minimum number of processed microarray samples for each organism"))
+        parser.add_argument("--qn-target",
+                            default=False,
+                            action='store_true',
+                            help=("Remove invalid QN targets"))
+        parser.add_argument("--compendias",
+                            default=False,
+                            action='store_true',
+                            help=("Remove invalid Compendias"))
+
+    def handle(self, *args, **options):
+        """ Takes care of removing invalid computational results. """
+        computational_result_ids = []
+
+        if options['compendias']:
+            computational_result_ids += remove_invalid_compendias(options['min'])
+
+        if options['qn_target']:
+            computational_result_ids += remove_invalid_qn_targets(options['min'])
+
+        if not computational_result_ids:
+            logger.info('Nothing removed. Use options --compendia or --qn-target to select which computational results to check.')
+
+        logger.info("Removing computational results with ids %s", str(computational_result_ids))
+
+        if not options['dry_run']:
+            # delete all invalid compendias from S3
+            compendias = ComputationalResult.objects.filter(id__in=computational_result_ids)
+            for computational_result in compendias:
+                computational_result.remove_computed_files_from_s3()
+            # delete all compendias
+            compendias.delete()
+
+
+def remove_invalid_qn_targets(min_samples):
+    qn_target_ids = []
+
+    for organism in Organism.object.filter(qn_target__isnull=False):
+        if not organism_can_have_qn_target(organism, min_samples):
+            # Remove all qn targets associated with this object
+            qn_target_ids += ComputationalResultAnnotation.objects\
+                .filter(data__is_qn=True, data__organism_id=organism.id)\
+                .values_list('result__id', flat=True)
+            logger.debug('Remove all QN targets for organism', organism=organism)
+
+    return qn_target_ids
+
+
+def remove_invalid_compendias(min_samples):
+    organism_with_compendia = _get_organisms_with_compendias()
+
+    computational_result_ids = []
+
+    for organism in organism_with_compendia:
+        if not organism_can_have_qn_target(organism, min_samples):
+            # Remove all compendias that are associated with organisms that can't have QN targets
+            computational_result_ids += ComputedFile.objects\
+                .filter(is_compendia=True, compendia_organism=organism)\
+                .values_list('result__id', flat=True)
+            logger.debug('Remove all Compendias for organism', organism=organism)
+
+    return computational_result_ids
+
+
+def _get_organisms_with_compendias():
+    organism_ids = ComputedFile.objects\
+        .filter(is_compendia=True)\
+        .values_list('compendia_organism__id', flat=True)\
+        .distinct()
+    return Organism.objects.filter(id__in=organism_ids)


Test patch: