Repository: AlexsLemonade/refinebio

Issue:
Make the dataset API simpler to use
### Context

Pairing with @kurtwheeler on the email validation code for the dataset endpoint this morning

### Problem or idea

@kurtwheeler pointed out that there is no good reason why we have to start the datset using a PUT request, except that is how we do it on the frontend. If everything is valid immediately, you should be able to create and dispatch a job in your initial POST request.

### Solution or next step

Refactor the dataset creation code so that it does not use a separate view for creating the dataset, and just validate POST requests the same way we validate PUT requests.


Discussion:

Resolving patch:
diff --git a/api/data_refinery_api/urls.py b/api/data_refinery_api/urls.py
--- a/api/data_refinery_api/urls.py
+++ b/api/data_refinery_api/urls.py
@@ -18,7 +18,6 @@
     ComputedFileDetailView,
     ComputedFileListView,
     CreateAPITokenView,
-    CreateDatasetView,
     DatasetView,
     DownloaderJobDetailView,
     DownloaderJobListView,
@@ -137,8 +136,12 @@ class AccessUser:
                     name="processors_detail",
                 ),
                 # Deliverables
-                url(r"^dataset/$", CreateDatasetView.as_view(), name="create_dataset"),
-                url(r"^dataset/(?P<id>[0-9a-f-]+)/$", DatasetView.as_view(), name="dataset"),
+                url(r"^dataset/$", DatasetView.as_view({"post": "create"}), name="create_dataset"),
+                url(
+                    r"^dataset/(?P<id>[0-9a-f-]+)/$",
+                    DatasetView.as_view({"get": "retrieve", "put": "update"}),
+                    name="dataset",
+                ),
                 url(r"^token/$", CreateAPITokenView.as_view(), name="token"),
                 url(r"^token/(?P<id>[0-9a-f-]+)/$", APITokenView.as_view(), name="token_id"),
                 # Jobs
diff --git a/api/data_refinery_api/views/__init__.py b/api/data_refinery_api/views/__init__.py
--- a/api/data_refinery_api/views/__init__.py
+++ b/api/data_refinery_api/views/__init__.py
@@ -8,7 +8,7 @@
     ComputationalResultListView,
 )
 from data_refinery_api.views.computed_file import ComputedFileDetailView, ComputedFileListView
-from data_refinery_api.views.dataset import CreateDatasetView, DatasetView
+from data_refinery_api.views.dataset import DatasetView
 from data_refinery_api.views.experiment import ExperimentDetailView, ExperimentListView
 from data_refinery_api.views.experiment_document import ExperimentDocumentView
 from data_refinery_api.views.institution import InstitutionListView
diff --git a/api/data_refinery_api/views/dataset.py b/api/data_refinery_api/views/dataset.py
--- a/api/data_refinery_api/views/dataset.py
+++ b/api/data_refinery_api/views/dataset.py
@@ -1,12 +1,12 @@
 ##
-# Contains CreateDatasetView and DatasetView
+# Contains DatasetView
 ##
 
 from collections import defaultdict
 
 from django.shortcuts import get_object_or_404
 from django.utils.decorators import method_decorator
-from rest_framework import filters, generics, serializers
+from rest_framework import filters, generics, mixins, serializers, viewsets
 
 from drf_yasg import openapi
 from drf_yasg.utils import swagger_auto_schema
@@ -88,22 +88,6 @@ def validate_dataset(data):
             )
 
 
-class CreateDatasetSerializer(serializers.ModelSerializer):
-    class Meta:
-        model = Dataset
-        fields = ("id", "data", "email_address", "email_ccdl_ok")
-
-    def validate(self, data):
-        """
-        Ensure this is something we want in our dataset.
-        """
-        try:
-            validate_dataset(data)
-        except Exception:
-            raise
-        return data
-
-
 class DatasetDetailsExperimentSerializer(serializers.ModelSerializer):
     """ This serializer contains all of the information about an experiment needed for the download
     page
@@ -142,6 +126,13 @@ def __init__(self, *args, **kwargs):
             if "token" not in kwargs["context"]:
                 self.fields.pop("download_url")
 
+    def create(self, validated_data):
+        # "start" isn't actually a field on the Dataset model, we just use it
+        # on the frontend to control when the dataset gets dispatched
+        if "start" in validated_data:
+            validated_data.pop("start")
+        return super(DatasetSerializer, self).create(validated_data)
+
     class Meta:
         model = Dataset
         fields = (
@@ -153,6 +144,8 @@ class Meta:
             "is_processed",
             "is_available",
             "has_email",
+            "email_address",
+            "email_ccdl_ok",
             "expires_on",
             "s3_bucket",
             "s3_key",
@@ -177,6 +170,8 @@ class Meta:
             "is_processing": {"read_only": True,},
             "is_processed": {"read_only": True,},
             "is_available": {"read_only": True,},
+            "email_address": {"required": False, "write_only": True},
+            "email_ccdl_ok": {"required": False, "write_only": True},
             "expires_on": {"read_only": True,},
             "s3_bucket": {"read_only": True,},
             "s3_key": {"read_only": True,},
@@ -232,15 +227,8 @@ def get_worker_version(self, obj):
             return None
 
 
-class CreateDatasetView(generics.CreateAPIView):
-    """ Creates and returns new Datasets. """
-
-    queryset = Dataset.objects.all()
-    serializer_class = CreateDatasetSerializer
-
-
 @method_decorator(
-    name="get",
+    name="retrieve",
     decorator=swagger_auto_schema(
         operation_description="View a single Dataset.",
         manual_parameters=[
@@ -254,10 +242,7 @@ class CreateDatasetView(generics.CreateAPIView):
     ),
 )
 @method_decorator(
-    name="patch", decorator=swagger_auto_schema(auto_schema=None)
-)  # partial updates not supported
-@method_decorator(
-    name="put",
+    name="update",
     decorator=swagger_auto_schema(
         operation_description="""
 Modify an existing Dataset.
@@ -285,7 +270,12 @@ class CreateDatasetView(generics.CreateAPIView):
 """
     ),
 )
-class DatasetView(generics.RetrieveUpdateAPIView):
+class DatasetView(
+    mixins.CreateModelMixin,
+    mixins.UpdateModelMixin,
+    mixins.RetrieveModelMixin,
+    viewsets.GenericViewSet,
+):
     """ View and modify a single Dataset. """
 
     queryset = Dataset.objects.all()
@@ -304,20 +294,24 @@ def get_serializer_context(self):
         except Exception:  # General APIToken.DoesNotExist or django.core.exceptions.ValidationError
             return serializer_context
 
-    def perform_update(self, serializer):
-        """ If `start` is set, fire off the job. Disables dataset data updates after that.
-        """
-        old_object = self.get_object()
-        old_data = old_object.data
-        old_aggregate = old_object.aggregate_by
-        already_processing = old_object.is_processing
-        new_data = serializer.validated_data
+    def validate_token(self):
+        # Make sure we have a valid activated token.
+        token_id = self.request.data.get("token_id", None)
 
-        qn_organisms = Organism.get_objects_with_qn_targets()
+        if not token_id:
+            token_id = self.request.META.get("HTTP_API_KEY", None)
+
+        try:
+            APIToken.objects.get(id=token_id, is_activated=True)
+        # General APIToken.DoesNotExist or django.core.exceptions.ValidationError
+        except Exception:
+            raise serializers.ValidationError("You must provide an active API token ID")
 
-        # We convert 'ALL' into the actual accession codes given
-        for key in new_data["data"].keys():
-            accessions = new_data["data"][key]
+    @staticmethod
+    def convert_ALL_to_accessions(data):
+        qn_organisms = Organism.get_objects_with_qn_targets()
+        for key in data["data"].keys():
+            accessions = data["data"][key]
             if accessions == ["ALL"]:
                 experiment = get_object_or_404(Experiment, accession_code=key)
 
@@ -326,92 +320,92 @@ def perform_update(self, serializer):
                         is_processed=True, organism__in=qn_organisms
                     ).values_list("accession_code", flat=True)
                 )
-                new_data["data"][key] = sample_codes
+                data["data"][key] = sample_codes
 
-        if old_object.is_processed:
-            raise serializers.ValidationError(
-                "You may not update Datasets which have already been processed"
+    def validate_email_address_is_nonempty(self):
+        """Check to make sure the email exists. We call this when getting ready to dispatch a dataset"""
+        supplied_email_address = self.request.data.get("email_address", None)
+        if supplied_email_address is None:
+            raise serializers.ValidationError("You must provide an email address.")
+
+    def dispatch_job(self, serializer, obj):
+        processor_job = ProcessorJob()
+        processor_job.pipeline_applied = "SMASHER"
+        processor_job.ram_amount = 4096
+        processor_job.save()
+
+        pjda = ProcessorJobDatasetAssociation()
+        pjda.processor_job = processor_job
+        pjda.dataset = obj
+        pjda.save()
+
+        job_sent = False
+
+        try:
+            # Hidden method of non-dispatching for testing purposes.
+            if not self.request.data.get("no_send_job", False):
+                job_sent = send_job(ProcessorPipeline.SMASHER, processor_job)
+            else:
+                # We didn't actually send it, but we also didn't want to.
+                job_sent = True
+        except Exception:
+            # job_sent is already false and the exception has
+            # already been logged by send_job, so nothing to
+            # do other than catch the exception.
+            pass
+
+        if not job_sent:
+            raise APIException(
+                "Unable to queue download job. Something has gone"
+                " wrong and we have been notified about it."
             )
-        if new_data.get("start"):
 
-            # Make sure we have a valid activated token.
-            token_id = self.request.data.get("token_id", None)
+        serializer.validated_data["is_processing"] = True
+        obj = serializer.save()
 
-            if not token_id:
-                token_id = self.request.META.get("HTTP_API_KEY", None)
+        # create a new dataset annotation with the information of this request
+        annotation = DatasetAnnotation()
+        annotation.dataset = obj
+        annotation.data = {
+            "start": True,
+            "ip": get_client_ip(self.request),
+            "user_agent": self.request.META.get("HTTP_USER_AGENT", None),
+        }
+        annotation.save()
 
-            try:
-                APIToken.objects.get(id=token_id, is_activated=True)
-            # General APIToken.DoesNotExist or django.core.exceptions.ValidationError
-            except Exception:
-                raise serializers.ValidationError("You must provide an active API token ID")
-
-            supplied_email_address = self.request.data.get("email_address", None)
-
-            if supplied_email_address is None:
-                raise serializers.ValidationError("You must provide an email address.")
-
-            email_ccdl_ok = self.request.data.get("email_ccdl_ok", False)
-
-            if not already_processing:
-                # Create and dispatch the new job.
-                processor_job = ProcessorJob()
-                processor_job.pipeline_applied = "SMASHER"
-                processor_job.ram_amount = 4096
-                processor_job.save()
-
-                pjda = ProcessorJobDatasetAssociation()
-                pjda.processor_job = processor_job
-                pjda.dataset = old_object
-                pjda.save()
-
-                job_sent = False
-
-                obj = serializer.save()
-                if obj.email_address != supplied_email_address:
-                    obj.email_address = supplied_email_address
-                    obj.save()
-                if email_ccdl_ok:
-                    obj.email_ccdl_ok = email_ccdl_ok
-                    obj.save()
-
-                try:
-                    # Hidden method of non-dispatching for testing purposes.
-                    if not self.request.data.get("no_send_job", False):
-                        job_sent = send_job(ProcessorPipeline.SMASHER, processor_job)
-                    else:
-                        # We didn't actually send it, but we also didn't want to.
-                        job_sent = True
-                except Exception:
-                    # job_sent is already false and the exception has
-                    # already been logged by send_job, so nothing to
-                    # do other than catch the exception.
-                    pass
-
-                if not job_sent:
-                    raise APIException(
-                        "Unable to queue download job. Something has gone"
-                        " wrong and we have been notified about it."
-                    )
-
-                serializer.validated_data["is_processing"] = True
-                obj = serializer.save()
-
-                # create a new dataset annotation with the information of this request
-                annotation = DatasetAnnotation()
-                annotation.dataset = old_object
-                annotation.data = {
-                    "start": True,
-                    "ip": get_client_ip(self.request),
-                    "user_agent": self.request.META.get("HTTP_USER_AGENT", None),
-                }
-                annotation.save()
-
-                return obj
+    def create_or_update(self, serializer):
+        """ If `start` is set, fire off the job. Otherwise just create/update the dataset"""
+        data = serializer.validated_data
+        DatasetView.convert_ALL_to_accessions(data)
 
+        if data.get("start"):
+            self.validate_token()
+            self.validate_email_address_is_nonempty()
+
+            obj = serializer.save()
+
+            self.dispatch_job(serializer, obj)
+        else:
+            serializer.save()
+
+    def perform_create(self, serializer):
+        # Since we are creating a new dataset, there is no way it is already processed
+        self.create_or_update(serializer)
+
+    def perform_update(self, serializer):
+        # Check to make sure we have not already processed the dataset
+        old_object = self.get_object()
+        if old_object.is_processed:
+            raise serializers.ValidationError(
+                "You may not update Datasets which have already been processed"
+            )
         # Don't allow critical data updates to jobs that have already been submitted,
         # but do allow email address updating.
-        if already_processing:
-            serializer.validated_data["data"] = old_data
-            serializer.validated_data["aggregate_by"] = old_aggregate
-        serializer.save()
+        elif old_object.is_processing:
+            self.validate_email_address_is_nonempty()
+
+            serializer.validated_data["data"] = old_object.data
+            serializer.validated_data["aggregate_by"] = old_object.aggregate_by
+            serializer.save()
+        else:
+            self.create_or_update(serializer)


Test patch:
diff --git a/api/data_refinery_api/tests.py b/api/data_refinery_api/tests.py
--- a/api/data_refinery_api/tests.py
+++ b/api/data_refinery_api/tests.py
@@ -500,6 +500,18 @@ def test_create_dataset_fails_without_email(self, mock_send_job):
         self.assertEqual(activated_token["is_activated"], True)
 
         # Good, except for missing email.
+        jdata = json.dumps(
+            {"start": True, "data": {"GSE123": ["789"]}, "token_id": activated_token["id"]}
+        )
+        response = self.client.post(
+            reverse("create_dataset", kwargs={"version": API_VERSION}),
+            jdata,
+            content_type="application/json",
+        )
+
+        self.assertEqual(response.status_code, 400)
+
+        # You should not have to provide an email until you set start=True
         jdata = json.dumps({"data": {"GSE123": ["789"]}})
         response = self.client.post(
             reverse("create_dataset", kwargs={"version": API_VERSION}),
@@ -509,7 +521,6 @@ def test_create_dataset_fails_without_email(self, mock_send_job):
 
         self.assertEqual(response.status_code, 201)
 
-        # Good, except for missing email.
         jdata = json.dumps(
             {"start": True, "data": {"GSE123": ["789"]}, "token_id": activated_token["id"]}
         )
@@ -824,6 +835,28 @@ def test_create_update_dataset(self, mock_send_job):
         self.assertEqual(ds.email_address, "trust@verify.com")
         self.assertTrue(ds.email_ccdl_ok)
 
+        # Test creating and starting a dataset in the same action
+        jdata = json.dumps(
+            {
+                "data": {"GSE123": ["789"]},
+                "start": True,
+                "no_send_job": True,
+                "email_address": "trust@verify.com",
+                "email_ccdl_ok": True,
+            }
+        )
+        response = self.client.post(
+            reverse("create_dataset", kwargs={"version": API_VERSION}),
+            jdata,
+            content_type="application/json",
+            HTTP_API_KEY=token_id,
+        )
+        self.assertEqual(response.json()["is_processing"], True)
+
+        ds = Dataset.objects.get(id=response.json()["id"])
+        self.assertEqual(ds.email_address, "trust@verify.com")
+        self.assertTrue(ds.email_ccdl_ok)
+
     def test_processed_samples_only(self):
         """ Don't return unprocessed samples """
         experiment = Experiment()