Repository: AlexsLemonade/refinebio

Issue:
Duplicate Sample Accessions Breaks Dataset State
### Context

@jashapiro was adding some samples to a dataset recently when he noticed on the client side he was no longer able to modify his dataset. After some investigation I noticed that the issue occurred when samples with the same accessions codes are present in multiple experiments. 

```python
  josh = Dataset.objects.get(id='5f3dd9da-e1fd-4ee6-8c07-bd97a83a0dd1')
  for key, value in josh.data.items():
    accessions.extend(value)
    
  len(accessions)
  # 1715

  samples = Sample.public_objects.filter(accession_code__in=accessions)
 
  samples.count()
  # 1709

  accession_set = Set(accessions)
  len(accession_set)
  # 1709

```

```python
for key, value in josh.data.items():
   items = set(value) & set(duplicates))
   if len(items) > 0:
     data[key] = items
 
   print(data)
   #  {
   #    "GSE98279": {
   #      "GSM2590437",
   #      "GSM2590436",
   #      "GSM2590433",
   #      "GSM2590434",
   #      "GSM2590432",
   #      "GSM2590435",
   #    },
   #    "GSE98384": {
   #      "GSM2590437",
   #      "GSM2590436",
   #      "GSM2590433",
   #      "GSM2590434",
   #      "GSM2590432",
   #      "GSM2590435",
   #    },
   # }
```
### Problem or idea

We should update this [line](https://github.com/AlexsLemonade/refinebio/blob/dev/api/data_refinery_api/views/dataset.py#L118) to either coerce the `accessions` list to a set or update the querset that defines `samples` to allow for duplicates.

We should also determine where in the logic that allows users to initially add the samples without failing this validation since the code should run before save. It only seems to be an issue after the duplicate sample accession codes are saved on the dataset instance and then another update is attempted.

### Solution or next step

We should update the code that checks if the dataset is in a valid state.



Discussion:
Probably worth also noting that this is probably coming up when there is a "SuperSeries" on GEO, where we catalog both the SuperSeries and the SubSeries.

So for the case below [GSE98384](https://www.ncbi.nlm.nih.gov/geo/query/acc.cgi?acc=GSE98384) is the superseries and  [GSE98279](https://www.ncbi.nlm.nih.gov/geo/query/acc.cgi?acc=GSE98279) is one of the subseries.

This also occurs in the case of GSE109403 & GSE109401, which I just narrowly avoiding both of to a dataset!



Resolving patch:
diff --git a/api/data_refinery_api/views/dataset.py b/api/data_refinery_api/views/dataset.py
--- a/api/data_refinery_api/views/dataset.py
+++ b/api/data_refinery_api/views/dataset.py
@@ -54,7 +54,7 @@ def experiment_has_downloadable_samples(experiment, quant_sf_only=False):
             results__computedfile__s3_bucket__isnull=False,
         )
 
-        if samples.count() == 0:
+        if not samples.exists():
             return False
 
     else:
@@ -75,60 +75,60 @@ def validate_dataset(data):
     if data.get("data") is None or type(data["data"]) != dict:
         raise InvalidData("`data` must be a dict of lists.")
 
-    if data.get("start") and len(data["data"]) == 0:
+    if data.get("start") and not data["data"]:
         raise InvalidData("`data` must contain at least one experiment.")
 
-    accessions = []
-    non_downloadable_experiments = []
+    accessions = set()
+    non_downloadable_experiments = set()
+
     for key, value in data["data"].items():
         if type(value) != list:
-            raise InvalidData("`data` must be a dict of lists. Problem with `" + str(key) + "`")
+            raise InvalidData(f"`data` must be a dict of lists. Problem with `{key}`")
 
-        if len(value) < 1:
+        if not value:
             raise InvalidData(
-                "`data` must be a dict of lists, each with one or more elements. Problem with `"
-                + str(key)
-                + "`"
+                "`data` must be a dict of lists, each with one or more elements. "
+                f"Problem with `{key}`"
             )
 
         if len(value) != len(set(value)):
-            raise InvalidData("Duplicate values detected in " + str(value))
+            raise InvalidData(f"Duplicate values detected in {value}")
 
         # If they want "ALL", just make sure that the experiment has at least one downloadable sample
         if value == ["ALL"]:
             if not experiment_has_downloadable_samples(
                 key, quant_sf_only=data.get("quant_sf_only", False)
             ):
-                non_downloadable_experiments.append(key)
+                non_downloadable_experiments.add(key)
 
         # Otherwise, we will check that all the samples they requested are downloadable
         else:
-            accessions.extend(value)
+            accessions.update(value)
 
-    if len(non_downloadable_experiments) != 0:
+    if non_downloadable_experiments:
         raise InvalidData(
             message="Experiment(s) in dataset have zero downloadable samples. See `details` for a full list",
             details=non_downloadable_experiments,
         )
 
-    if len(accessions) == 0:
+    if not accessions:
         return
 
     samples = Sample.public_objects.filter(accession_code__in=accessions)
     if samples.count() != len(accessions):
         raise InvalidData(
             message="Sample(s) in dataset do not exist on refine.bio. See `details` for a full list",
-            details=list(set(accessions) - set(s.accession_code for s in samples)),
+            details=list(accessions - set(s.accession_code for s in samples)),
         )
 
-    if data.get("quant_sf_only", False):
+    if data.get("quant_sf_only"):
         samples_without_quant_sf = samples.exclude(
             # Exclude samples that have at least one uploaded quant.sf file associated with them
             results__computedfile__filename="quant.sf",
             results__computedfile__s3_key__isnull=False,
             results__computedfile__s3_bucket__isnull=False,
         )
-        if samples_without_quant_sf.count() > 0:
+        if samples_without_quant_sf.exists():
             raise InvalidData(
                 message="Sample(s) in dataset are missing quant.sf files. See `details` for a full list",
                 details=[s.accession_code for s in samples_without_quant_sf],
@@ -136,7 +136,7 @@ def validate_dataset(data):
 
     else:
         unprocessed_samples = samples.exclude(is_processed=True)
-        if unprocessed_samples.count() > 0:
+        if unprocessed_samples.exists():
             raise InvalidData(
                 message="Non-downloadable sample(s) in dataset. See `details` for a full list",
                 details=[s.accession_code for s in unprocessed_samples],


Test patch:
diff --git a/api/tests/views/test_dataset.py b/api/tests/views/test_dataset.py
--- a/api/tests/views/test_dataset.py
+++ b/api/tests/views/test_dataset.py
@@ -348,7 +348,7 @@ def test_dataset_adding_non_downloadable_samples_fails(self):
             content_type="application/json",
         )
         self.assertIn(
-            "Sample(s) in dataset do not exist on refine",
+            "Sample(s) in dataset do not exist on refine.bio",
             response.json()["message"],
         )
         self.assertEqual(response.status_code, 400)
@@ -707,3 +707,63 @@ def test_create_dataset_notify_me(self):
 
         ds.refresh_from_db()
         self.assertFalse(ds.notify_me)
+
+    def test_add_dataset_multiple_experiment_sample(self):
+        response = self.client.post(
+            reverse("token", kwargs={"version": API_VERSION}),
+            json.dumps({"is_activated": True}),
+            content_type="application/json",
+        )
+        token_id = response.json()["id"]
+
+        response = self.client.post(
+            reverse("create_dataset", kwargs={"version": API_VERSION}),
+            json.dumps(
+                {
+                    "data": {
+                        "GSE000": ["789"],
+                    },
+                    "token_id": token_id,
+                }
+            ),
+            content_type="application/json",
+        )
+        self.assertEqual(response.status_code, 201)
+
+        data_set_id = response.json()["id"]
+        data = {
+            "GSE000": ["789"],
+            "GSE123": ["789"],
+        }
+        response = self.client.put(
+            reverse("dataset", kwargs={"id": data_set_id, "version": API_VERSION}),
+            json.dumps(
+                {
+                    "data": data,
+                    "token_id": token_id,
+                }
+            ),
+            content_type="application/json",
+        )
+        self.assertEqual(response.status_code, 200)
+
+        data_set = Dataset.objects.get(id=data_set_id)
+        self.assertEqual(data_set.data, data)
+
+        data = {
+            "GSE000": ["789"],
+        }
+        response = self.client.put(
+            reverse("dataset", kwargs={"id": data_set_id, "version": API_VERSION}),
+            json.dumps(
+                {
+                    "data": data,
+                    "token_id": token_id,
+                }
+            ),
+            content_type="application/json",
+        )
+        self.assertEqual(response.status_code, 200)
+
+        data_set.refresh_from_db()
+        self.assertEqual(data_set.data, data)