Repository: AlexsLemonade/refinebio

Issue:
`/search/` responses contain incorrect information
### Context

I found what was causing https://github.com/AlexsLemonade/refinebio-frontend/issues/955

### Problem or idea

Sometimes results returned from `/search/` contain incorrect information, which causes incorrect information like `No Species` to be displayed on the search page.

An example is GSE112517:

Hitting https://api.refine.bio/v1/search/?search=accession_code:GSE112517 gives
```
...
            "organism_names": [],
...
```
but hitting https://api.refine.bio/v1/experiments/GSE112517/ gives
```
...
            "organism_names": ["HOMO_SAPIENS"],
...
```

### Solution or next step

My first instinct is that the Elasticsearch database is out of sync with our database. One easy way to test this would be to check whether or not manually refreshing Elasticsearch solves the problem for this experiment.

I also think that this _could_ be happening because `organism_names` is a computed property, which might cause some weirdness. Surveying the experiment locally and seeing what a local instance of Elasticsearch returns would be a good way to test if this is true.


Discussion:
I tried surveying the experiment locally and I got the same result, so it looks like the issue is not just that our Elasticsearch instance is out of date. I will keep poking around and see what I find.
I did some digging, and the reason that this happens is because in Elasticsearch we get organism names in the following manner:
https://github.com/AlexsLemonade/refinebio/blob/8bba307678197c5f2aef06e35b8d7302f7aa3725/common/data_refinery_common/models/experiment.py#L213

but in the Experiment views we use:
https://github.com/AlexsLemonade/refinebio/blob/8bba307678197c5f2aef06e35b8d7302f7aa3725/api/data_refinery_api/views/experiment.py#L85
which covers _all_ organisms. I will have to change one of these to match the others, or figure out another solution.

Resolving patch:
diff --git a/api/data_refinery_api/views/experiment_document.py b/api/data_refinery_api/views/experiment_document.py
--- a/api/data_refinery_api/views/experiment_document.py
+++ b/api/data_refinery_api/views/experiment_document.py
@@ -141,6 +141,7 @@ class Meta(object):
             "platform_names",
             "platform_accession_codes",
             "organism_names",
+            "downloadable_organism_names",
             "pubmed_id",
             "num_total_samples",
             "num_processed_samples",
@@ -236,7 +237,7 @@ class Meta(object):
     ),
 )
 class ExperimentDocumentView(DocumentViewSet):
-    """ ElasticSearch powered experiment search. """
+    """ElasticSearch powered experiment search."""
 
     document = ExperimentDocument
     serializer_class = ExperimentDocumentSerializer
@@ -282,6 +283,7 @@ class ExperimentDocumentView(DocumentViewSet):
         "alternate_accession_code": "alternate_accession_code",
         "platform": "platform_accession_codes",
         "organism": "organism_names.raw",
+        "downloadable_organism": "downloadable_organism_names.raw",
         "num_processed_samples": {
             "field": "num_processed_samples",
             "lookups": [LOOKUP_FILTER_RANGE, LOOKUP_QUERY_IN, LOOKUP_QUERY_GT],
@@ -321,8 +323,8 @@ class ExperimentDocumentView(DocumentViewSet):
             "facet": TermsFacet,
             "enabled": True,  # These are enabled by default, which is more expensive but more simple.
         },
-        "organism_names": {
-            "field": "organism_names.raw",
+        "downloadable_organism_names": {
+            "field": "downloadable_organism_names.raw",
             "facet": TermsFacet,
             "enabled": True,
             "options": {"size": 999999},
diff --git a/common/data_refinery_common/models/documents.py b/common/data_refinery_common/models/documents.py
--- a/common/data_refinery_common/models/documents.py
+++ b/common/data_refinery_common/models/documents.py
@@ -35,8 +35,8 @@
 
 @experiment_index.doc_type
 class ExperimentDocument(Document):
-    """ Our Experiment ElasticSearch Document, which
-    corresponds to our Experiment model. """
+    """Our Experiment ElasticSearch Document, which
+    corresponds to our Experiment model."""
 
     # Keyword Fields
     title = fields.TextField(
@@ -57,6 +57,9 @@ class ExperimentDocument(Document):
     organism_names = fields.TextField(
         analyzer=html_strip_no_ngram, fielddata=True, fields={"raw": fields.KeywordField()}
     )
+    downloadable_organism_names = fields.TextField(
+        analyzer=html_strip_no_ngram, fielddata=True, fields={"raw": fields.KeywordField()}
+    )
     platform_names = fields.TextField(
         analyzer=standard_keyword, fielddata=True, fields={"raw": fields.TextField()}
     )
@@ -93,5 +96,5 @@ class Django:
         ]
 
     def get_queryset(self):
-        """ Override default queryset """
+        """Override default queryset"""
         return super(ExperimentDocument, self).get_queryset().order_by("id")
diff --git a/common/data_refinery_common/models/experiment.py b/common/data_refinery_common/models/experiment.py
--- a/common/data_refinery_common/models/experiment.py
+++ b/common/data_refinery_common/models/experiment.py
@@ -8,7 +8,7 @@
 
 
 class Experiment(models.Model):
-    """ An Experiment or Study """
+    """An Experiment or Study"""
 
     class Meta:
         db_table = "experiments"
@@ -66,7 +66,7 @@ def __str__(self):
     last_modified = models.DateTimeField(default=timezone.now)
 
     def save(self, *args, **kwargs):
-        """ On save, update timestamps """
+        """On save, update timestamps"""
         current_time = timezone.now()
         if not self.id:
             self.created_at = current_time
@@ -81,7 +81,7 @@ def save(self, *args, **kwargs):
         return super(Experiment, self).save(*args, **kwargs)
 
     def update_num_samples(self):
-        """ Update our cache values """
+        """Update our cache values"""
         aggregates = self.samples.aggregate(
             num_total_samples=Count("id"),
             num_processed_samples=Count("id", filter=Q(is_processed=True)),
@@ -95,7 +95,7 @@ def update_num_samples(self):
         self.save()
 
     def to_metadata_dict(self):
-        """ Render this Experiment as a dict """
+        """Render this Experiment as a dict"""
 
         metadata = {}
         metadata["title"] = self.title
@@ -128,7 +128,7 @@ def to_metadata_dict(self):
         return metadata
 
     def get_sample_keywords(self):
-        """ Get the human-readable name of all of the keywords that are defined
+        """Get the human-readable name of all of the keywords that are defined
         on at least one sample
         """
         keywords = set()
@@ -139,7 +139,7 @@ def get_sample_keywords(self):
         return list(keywords)
 
     def get_sample_metadata_fields(self):
-        """ Get all metadata fields that are non-empty for at least one sample in the experiment.
+        """Get all metadata fields that are non-empty for at least one sample in the experiment.
         See https://github.com/AlexsLemonade/refinebio-frontend/issues/211 for why this is needed.
         """
         fields = []
@@ -178,28 +178,25 @@ def update_platform_names(self):
         self.platform_accession_codes = self.get_platform_accession_codes()
 
     def get_sample_technologies(self):
-        """ Get a list of unique technologies for all of the associated samples
-        """
+        """Get a list of unique technologies for all of the associated samples"""
         return list(set([sample.technology for sample in self.samples.all()]))
 
     def get_platform_names(self):
-        """ Get a list of unique platforms for all of the associated samples
-        """
+        """Get a list of unique platforms for all of the associated samples"""
         return list(set([sample.platform_name for sample in self.samples.all()]))
 
     def get_platform_accession_codes(self):
-        """ Get a list of unique platforms for all of the associated samples
-        """
+        """Get a list of unique platforms for all of the associated samples"""
         return list(set([sample.platform_accession_code for sample in self.samples.all()]))
 
     @property
     def platforms(self):
-        """ Returns a list of related pipelines """
+        """Returns a list of related pipelines"""
         return list(set([sample.platform_name for sample in self.samples.all()]))
 
     @property
     def pretty_platforms(self):
-        """ Returns a prettified list of related pipelines """
+        """Returns a prettified list of related pipelines"""
         return list(set([sample.pretty_platform for sample in self.samples.all()]))
 
     @property
@@ -209,13 +206,21 @@ def processed_samples(self):
         )
 
     @property
-    def organism_names(self):
-        """ Get a list of unique organism names that has at least one downloadable sample """
+    def downloadable_organism_names(self):
+        """Get a list of unique organism names that has at least one downloadable sample"""
         result = (
             self.samples.filter(is_processed=True, organism__qn_target__isnull=False)
             .values_list("organism__name", flat=True)
             .distinct()
         )
+
+        return list(result)
+
+    @property
+    def organism_names(self):
+        """Get a list of unique organism names"""
+        result = self.samples.values_list("organism__name", flat=True).distinct()
+
         return list(result)
 
     @property


Test patch:
diff --git a/api/data_refinery_api/test/test_search.py b/api/data_refinery_api/test/test_search.py
--- a/api/data_refinery_api/test/test_search.py
+++ b/api/data_refinery_api/test/test_search.py
@@ -164,9 +164,12 @@ def test_es_endpoint(self):
         self.assertEqual(response.json()["facets"]["technology"]["microarray"], 1)
         self.assertEqual(response.json()["facets"]["technology"]["rna-seq"], 0)
         self.assertEqual(
-            list(response.json()["facets"]["organism_names"].keys()), [ECOLI_STRAIN_NAME]
+            list(response.json()["facets"]["downloadable_organism_names"].keys()),
+            [ECOLI_STRAIN_NAME],
+        )
+        self.assertEqual(
+            response.json()["facets"]["downloadable_organism_names"][ECOLI_STRAIN_NAME], 1
         )
-        self.assertEqual(response.json()["facets"]["organism_names"][ECOLI_STRAIN_NAME], 1)
 
         # Basic Search
         response = self.client.get(