Repository: AlexsLemonade/refinebio

Issue:
Validate that samples added to the dataset are downloadable
### Context

We allow datasets like [this](https://www.refine.bio/dataset/98e449b3-b485-4786-8529-79dc50e4cd5b) to be created.

### Problem or idea

This dataset has *at least one experiment* and it also has *at least one sample per experiment*, but it still isn't valid because the samples are not downloadable.

### Solution or next step

The API should also verify that all the samples added to the dataset are downloadable.



Discussion:

Resolving patch:
diff --git a/api/data_refinery_api/serializers.py b/api/data_refinery_api/serializers.py
--- a/api/data_refinery_api/serializers.py
+++ b/api/data_refinery_api/serializers.py
@@ -592,35 +592,59 @@ class Meta:
 
 
 def validate_dataset(data):
-    """ Basic dataset validation. Currently only checks formatting, not values. """
-    if data.get("data") is not None:
-        if type(data["data"]) != dict:
-            raise serializers.ValidationError("`data` must be a dict of lists.")
+    """
+    Dataset validation. Each experiment should always have at least one
+    sample, all samples should be downloadable, and when starting the smasher
+    there should be at least one experiment.
+    """
+    if data.get("data") is None or type(data["data"]) != dict:
+        raise serializers.ValidationError("`data` must be a dict of lists.")
 
-        if data.get("start") and len(data["data"]) == 0:
-            raise serializers.ValidationError("`data` must contain at least one experiment..")
+    if data.get("start") and len(data["data"]) == 0:
+        raise serializers.ValidationError("`data` must contain at least one experiment..")
 
-        for key, value in data["data"].items():
-            if type(value) != list:
-                raise serializers.ValidationError(
-                    "`data` must be a dict of lists. Problem with `" + str(key) + "`"
-                )
+    accessions = []
+    for key, value in data["data"].items():
+        if type(value) != list:
+            raise serializers.ValidationError(
+                "`data` must be a dict of lists. Problem with `" + str(key) + "`"
+            )
 
-            if len(value) < 1:
-                raise serializers.ValidationError(
-                    "`data` must be a dict of lists, each with one or more elements. Problem with `"
-                    + str(key)
-                    + "`"
-                )
+        if len(value) < 1:
+            raise serializers.ValidationError(
+                "`data` must be a dict of lists, each with one or more elements. Problem with `"
+                + str(key)
+                + "`"
+            )
 
+        try:
+            if len(value) != len(set(value)):
+                raise serializers.ValidationError("Duplicate values detected in " + str(value))
+        except Exception as e:
+            raise serializers.ValidationError("Received bad dataset data: " + str(e))
+
+        # If they want "ALL", just make sure that the experiment has at least one downloadable sample
+        if value == ["ALL"]:
             try:
-                if len(value) != len(set(value)):
-                    raise serializers.ValidationError("Duplicate values detected in " + str(value))
+                experiment = Experiment.processed_public_objects.get(accession_code=key)
             except Exception as e:
-                raise serializers.ValidationError("Received bad dataset data: " + str(e))
+                raise serializers.ValidationError(
+                    "Experiment " + key + " does not have at least one downloadable sample"
+                )
+        # Otherwise, we will check that all the samples they requested are downloadable
+        else:
+            accessions.extend(value)
 
-    else:
-        raise serializers.ValidationError("`data` must be a dict of lists.")
+    if len(accessions) > 0:
+        unprocessed_samples = Sample.public_objects.filter(
+            accession_code__in=accessions, is_processed=False
+        )
+        if unprocessed_samples.count() > 0:
+            raise serializers.ValidationError(
+                "Non-downloadable sample(s) '"
+                + ", ".join([s.accession_code for s in unprocessed_samples])
+                + "' in dataset"
+            )
 
 
 class CreateDatasetSerializer(serializers.ModelSerializer):


Test patch:
diff --git a/api/data_refinery_api/tests.py b/api/data_refinery_api/tests.py
--- a/api/data_refinery_api/tests.py
+++ b/api/data_refinery_api/tests.py
@@ -440,7 +440,7 @@ def test_create_dataset_fails_without_email(self, mock_send_job):
         self.assertEqual(activated_token["is_activated"], True)
 
         # Good, except for missing email.
-        jdata = json.dumps({"data": {"A": ["B"]}})
+        jdata = json.dumps({"data": {"GSE123": ["789"]}})
         response = self.client.post(
             reverse("create_dataset", kwargs={"version": API_VERSION}),
             jdata,
@@ -505,13 +505,10 @@ def test_starting_dataset_fails_without_experiments(self, mock_send_job):
         )
         self.assertEqual(response.status_code, 201)
 
-        data["start"] = True
-        data["data"] = {"A": ["B"]}
-        data["token_id"] = token_id
         data = {
             **data,
             "start": True,
-            "data": {"A": ["B"]},
+            "data": {"GSE123": ["789"]},
             "token_id": token_id,
             "no_send_job": True,
         }
@@ -522,6 +519,58 @@ def test_starting_dataset_fails_without_experiments(self, mock_send_job):
         )
         self.assertEqual(response.status_code, 200)
 
+    def test_dataset_adding_non_downloadable_samples_fails(self):
+        sample1 = Sample()
+        sample1.title = "456"
+        sample1.accession_code = "456"
+        sample1.platform_name = "AFFY"
+        sample1.is_processed = False
+        sample1.organism = self.homo_sapiens
+        sample1.save()
+
+        experiment_sample_association = ExperimentSampleAssociation()
+        experiment_sample_association.sample = sample1
+        experiment_sample_association.experiment = self.experiment
+        experiment_sample_association.save()
+
+        # Get a token first
+        response = self.client.post(
+            reverse("token", kwargs={"version": API_VERSION}), content_type="application/json"
+        )
+        token = response.json()
+        token["is_activated"] = True
+        token_id = token["id"]
+        response = self.client.put(
+            reverse("token_id", kwargs={"id": token_id, "version": API_VERSION}),
+            json.dumps(token),
+            content_type="application/json",
+        )
+
+        activated_token = response.json()
+        self.assertEqual(activated_token["id"], token_id)
+        self.assertEqual(activated_token["is_activated"], True)
+
+        # Bad, 456 is not processed
+        jdata = json.dumps({"email_address": "baz@gmail.com", "data": {"GSE123": ["456"]}})
+        response = self.client.post(
+            reverse("create_dataset", kwargs={"version": API_VERSION}),
+            jdata,
+            content_type="application/json",
+        )
+        self.assertEqual(response.status_code, 400)
+        self.assertEqual(
+            response.json()["non_field_errors"][0], "Non-downloadable sample(s) '456' in dataset"
+        )
+
+        # Good, 789 is processed
+        jdata = json.dumps({"email_address": "baz@gmail.com", "data": {"GSE123": ["789"]}})
+        response = self.client.post(
+            reverse("create_dataset", kwargs={"version": API_VERSION}),
+            jdata,
+            content_type="application/json",
+        )
+        self.assertEqual(response.status_code, 201)
+
     @patch("data_refinery_common.message_queue.send_job")
     def test_create_update_dataset(self, mock_send_job):
 
@@ -542,7 +591,7 @@ def test_create_update_dataset(self, mock_send_job):
         self.assertEqual(activated_token["id"], token_id)
         self.assertEqual(activated_token["is_activated"], True)
         # Good
-        jdata = json.dumps({"email_address": "baz@gmail.com", "data": {"A": ["B"]}})
+        jdata = json.dumps({"email_address": "baz@gmail.com", "data": {"GSE123": ["789"]}})
         response = self.client.post(
             reverse("create_dataset", kwargs={"version": API_VERSION}),
             jdata,
@@ -558,10 +607,12 @@ def test_create_update_dataset(self, mock_send_job):
         )
         self.assertEqual(response.json()["id"], good_id)
         self.assertEqual(response.json()["data"], json.loads(jdata)["data"])
-        self.assertEqual(response.json()["data"]["A"], ["B"])
+        self.assertEqual(response.json()["data"]["GSE123"], ["789"])
 
         # Bad (Duplicates)
-        jdata = json.dumps({"email_address": "baz@gmail.com", "data": {"A": ["B", "B", "B"]}})
+        jdata = json.dumps(
+            {"email_address": "baz@gmail.com", "data": {"GSE123": ["789", "789", "789"]}}
+        )
         response = self.client.post(
             reverse("create_dataset", kwargs={"version": API_VERSION}),
             jdata,
@@ -571,7 +622,7 @@ def test_create_update_dataset(self, mock_send_job):
         self.assertEqual(response.status_code, 400)
 
         # Bad (Empty Experiment)
-        jdata = json.dumps({"email_address": "baz@gmail.com", "data": {"A": []}})
+        jdata = json.dumps({"email_address": "baz@gmail.com", "data": {"GSE123": []}})
         response = self.client.post(
             reverse("create_dataset", kwargs={"version": API_VERSION}),
             jdata,
@@ -594,7 +645,7 @@ def test_create_update_dataset(self, mock_send_job):
         self.assertEqual(response.json()["data"]["GSE123"], ["789"])
 
         # Update
-        jdata = json.dumps({"email_address": "baz@gmail.com", "data": {"A": ["C"]}})
+        jdata = json.dumps({"email_address": "baz@gmail.com", "data": {"GSE123": ["789"]}})
         response = self.client.put(
             reverse("dataset", kwargs={"id": good_id, "version": API_VERSION}),
             jdata,
@@ -603,19 +654,21 @@ def test_create_update_dataset(self, mock_send_job):
         self.assertEqual(response.status_code, 200)
         self.assertEqual(response.json()["id"], good_id)
         self.assertEqual(response.json()["data"], json.loads(jdata)["data"])
-        self.assertEqual(response.json()["data"]["A"], ["C"])
+        self.assertEqual(response.json()["data"]["GSE123"], ["789"])
 
         # Can't update if started
         dataset = Dataset.objects.get(id=good_id)
         dataset.is_processing = True
         dataset.save()
-        jdata = json.dumps({"email_address": "baz@gmail.com", "data": {"A": ["D"]}})
+        jdata = json.dumps({"email_address": "baz@gmail.com", "data": {"GSE123": ["123"]}})
         response = self.client.put(
             reverse("dataset", kwargs={"id": good_id, "version": API_VERSION}),
             jdata,
             content_type="application/json",
         )
-        self.assertNotEqual(response.json()["data"]["A"], ["D"])
+        # The request should succeed, but the dataset should not be modified
+        self.assertEqual(response.status_code, 200)
+        self.assertNotEqual(response.json()["data"]["GSE123"], ["123"])
 
         # Bad
         jdata = json.dumps({"email_address": "baz@gmail.com", "data": 123})
@@ -635,7 +688,7 @@ def test_create_update_dataset(self, mock_send_job):
         jdata = json.dumps(
             {
                 "email_address": "baz@gmail.com",
-                "data": {"A": ["D"]},
+                "data": {"GSE123": ["789"]},
                 "start": True,
                 "no_send_job": True,
                 "token_id": "HEYO",
@@ -650,7 +703,7 @@ def test_create_update_dataset(self, mock_send_job):
 
         jdata = json.dumps(
             {
-                "data": {"A": ["D"]},
+                "data": {"GSE123": ["789"]},
                 "start": True,
                 "no_send_job": True,
                 "token_id": token_id,
@@ -679,7 +732,7 @@ def test_create_update_dataset(self, mock_send_job):
 
         jdata = json.dumps(
             {
-                "data": {"A": ["D"]},
+                "data": {"GSE123": ["789"]},
                 "start": True,
                 "no_send_job": True,
                 "email_address": "trust@verify.com",
@@ -785,6 +838,7 @@ def test_dataset_stats(self):
         sample1.title = "1"
         sample1.accession_code = "1"
         sample1.platform_name = "AFFY"
+        sample1.is_processed = True
         sample1.organism = self.homo_sapiens
         sample1.save()
 
@@ -792,6 +846,7 @@ def test_dataset_stats(self):
         sample2.title = "2"
         sample2.accession_code = "2"
         sample2.platform_name = "ILLUMINA"
+        sample2.is_processed = True
         sample2.organism = gallus_gallus
         sample2.save()
 
@@ -799,6 +854,7 @@ def test_dataset_stats(self):
         sample3.title = "3"
         sample3.accession_code = "3"
         sample3.platform_name = "ILLUMINA"
+        sample3.is_processed = True
         sample3.organism = gallus_gallus
         sample3.save()