Repository: Parsl/parsl

Issue:
scale out mechanism in HTEX not compatible with kubernetes provider
Currently, the `scale_out` mechanism in HTEX is to scale out block by block, as in https://github.com/Parsl/parsl/blob/a00acdd411c44777a01d79b0f9cde2cb51e27aff/parsl/executors/high_throughput/executor.py#L480-L482

This is conflicted with the mechanism in kubernetes provider. In kube provider, the number of blocks refer to the number of pods, and it creates the blocks (pods) directly, as in 
https://github.com/Parsl/parsl/blob/a00acdd411c44777a01d79b0f9cde2cb51e27aff/parsl/providers/kubernetes/kube.py#L126-L131

Need to fix to make them compatible.


Discussion:
I am going to add a check in the executor to let kubernetes provider skip this loop below:
https://github.com/Parsl/parsl/blob/a00acdd411c44777a01d79b0f9cde2cb51e27aff/parsl/executors/high_throughput/executor.py#L480-L482

 Any better suggestion for this issue?
@zhuozhaoli so that scale_out does nothing in that case?
> @ZhuozhaoLi so that scale_out does nothing in that case?

@benclifford So the scale_out only does `self.provider.submit(self.launch_cmd, 1, 1)` once, rather than `blocks` times for kubernetes provider -- should be `break the loop after once`.
Some questions that spring to mind:

Who calls scale_out?

Are they going to be happy that they didn't get the number of blocks they ask for?

What happens if they ask to scale out more blocks by calling scale_out repeatedly because you didn't scale out the right number of blocks?

Should this code throw a ValueError if it cannot do what it was asked to do rather than silently doing something else?

Architecturally it's horrible to put in class type tests all over the place - the suggested change isn't going to work for the not yet implemented SimilarToKubernetesProvider, for example.
The result of a discussion between myself and @ZhuozhaoLi on slack was that a better representation of blocks in kubernetes might be: each block is a kubernetes pod, managed directly via parsl, rather than via a kubernetes `deployment` which doesn't allow pod management in the way that our block management is headed.


Resolving patch:
diff --git a/parsl/providers/kubernetes/kube.py b/parsl/providers/kubernetes/kube.py
--- a/parsl/providers/kubernetes/kube.py
+++ b/parsl/providers/kubernetes/kube.py
@@ -44,6 +44,9 @@ class KubernetesProvider(ExecutionProvider, RepresentationMixin):
         Command to be run first for the workers, such as `python start.py`.
     secret : str
         Docker secret to use to pull images
+    pod_name : str
+        The name for the pod, will be appended with a timestamp.
+        Default is None, meaning parsl automatically names the pod.
     user_id : str
         Unix user id to run the container as.
     group_id : str
@@ -65,7 +68,7 @@ def __init__(self,
                  max_blocks=10,
                  parallelism=1,
                  worker_init="",
-                 deployment_name=None,
+                 pod_name=None,
                  user_id=None,
                  group_id=None,
                  run_as_non_root=False,
@@ -86,13 +89,13 @@ def __init__(self,
         self.parallelism = parallelism
         self.worker_init = worker_init
         self.secret = secret
-        self.deployment_name = deployment_name
+        self.pod_name = pod_name
         self.user_id = user_id
         self.group_id = group_id
         self.run_as_non_root = run_as_non_root
         self.persistent_volumes = persistent_volumes
 
-        self.kube_client = client.ExtensionsV1beta1Api()
+        self.kube_client = client.CoreV1Api()
 
         # Dictionary that keeps track of jobs, keyed on job_id
         self.resources = {}
@@ -110,31 +113,29 @@ def submit(self, cmd_string, blocksize, tasks_per_node, job_name="parsl"):
              - None: At capacity, cannot provision more
              - job_id: (string) Identifier for the job
         """
-        if not self.resources:
-            cur_timestamp = str(time.time() * 1000).split(".")[0]
-            job_name = "{0}-{1}".format(job_name, cur_timestamp)
-
-            if not self.deployment_name:
-                deployment_name = '{}-deployment'.format(job_name)
-            else:
-                deployment_name = '{}-{}-deployment'.format(self.deployment_name,
-                                                            cur_timestamp)
-
-            formatted_cmd = template_string.format(command=cmd_string,
-                                                   worker_init=self.worker_init)
-
-            self.deployment_obj = self._create_deployment_object(job_name,
-                                                                 self.image,
-                                                                 deployment_name,
-                                                                 cmd_string=formatted_cmd,
-                                                                 replicas=self.init_blocks,
-                                                                 volumes=self.persistent_volumes)
-            logger.debug("Deployment name :{}".format(deployment_name))
-            self._create_deployment(self.deployment_obj)
-            self.resources[deployment_name] = {'status': 'RUNNING',
-                                               'pods': self.init_blocks}
-
-        return deployment_name
+
+        cur_timestamp = str(time.time() * 1000).split(".")[0]
+        job_name = "{0}-{1}".format(job_name, cur_timestamp)
+
+        if not self.pod_name:
+            pod_name = '{}'.format(job_name)
+        else:
+            pod_name = '{}-{}'.format(self.pod_name,
+                                      cur_timestamp)
+
+        formatted_cmd = template_string.format(command=cmd_string,
+                                               worker_init=self.worker_init)
+
+        logger.debug("Pod name :{}".format(pod_name))
+        self._create_pod(image=self.image,
+                         pod_name=pod_name,
+                         job_name=job_name,
+                         cmd_string=formatted_cmd,
+                         volumes=self.persistent_volumes)
+        self.resources[pod_name] = {'status': 'RUNNING',
+                                    'pods': blocksize}
+
+        return pod_name
 
     def status(self, job_ids):
         """ Get the status of a list of jobs identified by the job identifiers
@@ -161,9 +162,10 @@ def cancel(self, job_ids):
         for job in job_ids:
             logger.debug("Terminating job/proc_id: {0}".format(job))
             # Here we are assuming that for local, the job_ids are the process id's
-            self._delete_deployment(job)
+            self._delete_pod(job)
 
             self.resources[job]['status'] = 'CANCELLED'
+            del self.resources[job]
         rets = [True for i in job_ids]
 
         return rets
@@ -181,26 +183,24 @@ def _status(self):
         return jobs_ids
         # do something to get the deployment's status
 
-    def _create_deployment_object(self, job_name, job_image,
-                                  deployment_name, port=80,
-                                  replicas=1,
-                                  cmd_string=None,
-                                  engine_json_file='~/.ipython/profile_default/security/ipcontroller-engine.json',
-                                  engine_dir='.',
-                                  volumes=[]):
-        """ Create a kubernetes deployment for the job.
+    def _create_pod(self,
+                    image,
+                    pod_name,
+                    job_name,
+                    port=80,
+                    cmd_string=None,
+                    volumes=[]):
+        """ Create a kubernetes pod for the job.
         Args:
-              - job_name (string) : Name of the job and deployment
-              - job_image (string) : Docker image to launch
+              - image (string) : Docker image to launch
+              - pod_name (string) : Name of the pod
+              - job_name (string) : App label
         KWargs:
              - port (integer) : Container port
-             - replicas : Number of replica containers to maintain
         Returns:
-              - True: The deployment object to launch
+              - None
         """
 
-        # sorry, quick hack that doesn't pass this stuff through to test it works.
-        # TODO it also doesn't only add what is set :(
         security_context = None
         if self.user_id and self.group_id:
             security_context = client.V1SecurityContext(run_as_group=self.group_id,
@@ -210,34 +210,24 @@ def _create_deployment_object(self, job_name, job_image,
         # Create the enviornment variables and command to initiate IPP
         environment_vars = client.V1EnvVar(name="TEST", value="SOME DATA")
 
-        launch_args = ["-c", "{0}; /app/deploy.sh;".format(cmd_string)]
+        launch_args = ["-c", "{0};".format(cmd_string)]
 
         volume_mounts = []
         # Create mount paths for the volumes
         for volume in volumes:
             volume_mounts.append(client.V1VolumeMount(mount_path=volume[1],
                                                       name=volume[0]))
-        # Configureate Pod template container
-        container = None
-        if security_context:
-            container = client.V1Container(
-                name=job_name,
-                image=job_image,
-                ports=[client.V1ContainerPort(container_port=port)],
-                volume_mounts=volume_mounts,
-                command=['/bin/bash'],
-                args=launch_args,
-                env=[environment_vars],
-                security_context=security_context)
-        else:
-            container = client.V1Container(
-                name=job_name,
-                image=job_image,
-                ports=[client.V1ContainerPort(container_port=port)],
-                volume_mounts=volume_mounts,
-                command=['/bin/bash'],
-                args=launch_args,
-                env=[environment_vars])
+        # Configure Pod template container
+        container = client.V1Container(
+            name=pod_name,
+            image=image,
+            ports=[client.V1ContainerPort(container_port=port)],
+            volume_mounts=volume_mounts,
+            command=['/bin/bash'],
+            args=launch_args,
+            env=[environment_vars],
+            security_context=security_context)
+
         # Create a secret to enable pulling images from secure repositories
         secret = None
         if self.secret:
@@ -250,52 +240,29 @@ def _create_deployment_object(self, job_name, job_image,
                                                persistent_volume_claim=client.V1PersistentVolumeClaimVolumeSource(
                                                    claim_name=volume[0])))
 
-        # Create and configurate a spec section
-        template = client.V1PodTemplateSpec(
-            metadata=client.V1ObjectMeta(labels={"app": job_name}),
-            spec=client.V1PodSpec(containers=[container],
-                                  image_pull_secrets=[secret],
-                                  volumes=volume_defs
-                                  ))
+        metadata = client.V1ObjectMeta(name=pod_name,
+                                       labels={"app": job_name})
+        spec = client.V1PodSpec(containers=[container],
+                                image_pull_secrets=[secret],
+                                volumes=volume_defs
+                                )
 
-        # Create the specification of deployment
-        spec = client.ExtensionsV1beta1DeploymentSpec(replicas=replicas,
-                                                      template=template)
+        pod = client.V1Pod(spec=spec, metadata=metadata)
+        api_response = self.kube_client.create_namespaced_pod(namespace=self.namespace,
+                                                              body=pod)
+        logger.debug("Pod created. status='{0}'".format(str(api_response.status)))
 
-        # Instantiate the deployment object
-        deployment = client.ExtensionsV1beta1Deployment(
-            api_version="extensions/v1beta1",
-            kind="Deployment",
-            metadata=client.V1ObjectMeta(name=deployment_name),
-            spec=spec)
+    def _delete_pod(self, pod_name):
+        """Delete a pod"""
 
-        return deployment
-
-    def _create_deployment(self, deployment):
-        """ Create the kubernetes deployment """
-
-        api_response = self.kube_client.create_namespaced_deployment(
-            body=deployment,
-            namespace=self.namespace)
-
-        logger.debug("Deployment created. status='{0}'".format(str(api_response.status)))
-
-    def _delete_deployment(self, deployment_name):
-        """ Delete deployment """
-
-        api_response = self.kube_client.delete_namespaced_deployment(
-            name=deployment_name,
-            namespace=self.namespace,
-            body=client.V1DeleteOptions(
-                propagation_policy='Foreground',
-                grace_period_seconds=5))
-
-        logger.debug("Deployment deleted. status='{0}'".format(
-            str(api_response.status)))
+        api_response = self.kube_client.delete_namespaced_pod(name=pod_name,
+                                                              namespace=self.namespace,
+                                                              body=client.V1DeleteOptions())
+        logger.debug("Pod deleted. status='{0}'".format(str(api_response.status)))
 
     @property
     def scaling_enabled(self):
-        return False
+        return True
 
     @property
     def channels_required(self):


Test patch: