Repository: Parsl/parsl

Issue:
Update Kubernetes provider to new config system
Reported by @ryanchard.

We forgot to update the Kubernetes provider with the 0.6 changes.


Discussion:

Resolving patch:
diff --git a/parsl/providers/kubernetes/kube.py b/parsl/providers/kubernetes/kube.py
--- a/parsl/providers/kubernetes/kube.py
+++ b/parsl/providers/kubernetes/kube.py
@@ -1,86 +1,107 @@
 import logging
 import time
+from parsl.providers.kubernetes.template import template_string
+
 logger = logging.getLogger(__name__)
 
+from parsl.providers.error import *
 from parsl.providers.provider_base import ExecutionProvider
-from parsl.providers.kubernetes.template import template_string
-from parsl.providers.error import OptionalModuleMissing
-
+from parsl.utils import RepresentationMixin
 
 try:
-    from kubernetes import client
-    from kubernetes import config as kube_config
+    from kubernetes import client, config
     _kubernetes_enabled = True
 except (ImportError, NameError, FileNotFoundError):
     _kubernetes_enabled = False
 
 
-class KubernetesProvider(ExecutionProvider):
-    """ Kubernetes execution provider:
-
-        TODO: put in a config
+class KubernetesProvider(ExecutionProvider, RepresentationMixin):
+    """ Kubernetes execution provider
+    Parameters
+    ----------
+    namespace : str
+        Kubernetes namespace to create deployments.
+    image : str
+        Docker image to use in the deployment.
+    channel : Channel
+        Channel for accessing this provider. Possible channels include
+        :class:`~parsl.channels.LocalChannel` (the default),
+        :class:`~parsl.channels.SSHChannel`, or
+        :class:`~parsl.channels.SSHInteractiveLoginChannel`.
+    tasks_per_node : int
+        Tasks to run per node.
+    nodes_per_block : int
+        Nodes to provision per block.
+    init_blocks : int
+        Number of blocks to provision at the start of the run. Default is 1.
+    min_blocks : int
+        Minimum number of blocks to maintain.
+    max_blocks : int
+        Maximum number of blocks to maintain.
+    parallelism : float
+        Ratio of provisioned task slots to active tasks. A parallelism value of 1 represents aggressive
+        scaling where as many resources as possible are used; parallelism close to 0 represents
+        the opposite situation in which as few resources as possible (i.e., min_blocks) are used.
+    secret : str
+        Docker secret to use to pull images
+    user_id : str
+        Unix user id to run the container as.
+    group_id : str
+        Unix group id to run the container as.
+    run_as_non_root : bool
+        Run as non-root (True) or run as root (False).
     """
 
-    def __repr__(self):
-        return "<Kubernetes Execution Provider for site:{0}>".format(self.sitename)
-
-    def __init__(self, config, channel=None):
-        """ Initialize the Kubernetes execution provider class
-
-        Args:
-             - Config (dict): Dictionary with all the config options.
-
-        KWargs :
-             - channel (channel object) : default=None A channel object
-        """
-        kube_config.load_kube_config()
-
-        self.channel = channel
-
+    def __init__(self,
+                 image,
+                 namespace='default',
+                 channel=None,
+                 tasks_per_node=1,
+                 nodes_per_block=1,
+                 init_blocks=4,
+                 min_blocks=0,
+                 max_blocks=10,
+                 parallelism=1,
+                 overrides="",
+                 user_id=None,
+                 group_id=None,
+                 run_as_non_root=False,
+                 secret=None):
         if not _kubernetes_enabled:
             raise OptionalModuleMissing(['kubernetes'],
                                         "Kubernetes provider requires kubernetes module and config.")
+        config.load_kube_config()
 
-        self.kube_client = client.ExtensionsV1beta1Api()
-
-        self.config = config
-        self.sitename = self.config['site']
-        self.namespace = self.config['execution']['namespace']
-        self.image = self.config['execution']['image']
-
-        self.init_blocks = self.config["execution"]["block"]["initBlocks"]
-        self.min_blocks = self.config["execution"]["block"]["minBlocks"]
-        self.max_blocks = self.config["execution"]["block"]["maxBlocks"]
-
-        self.user_id = None
-        self.group_id = None
-        self.run_as_non_root = None
-        if 'security' in self.config['execution']:
-            self.user_id = self.config["execution"]['security']["user_id"]
-            self.group_id = self.config["execution"]['security']["group_id"]
-            self.run_as_non_root = self.config["execution"]['security']["run_as_non_root"]
+        self.namespace = namespace
+        self.image = image
+        self.channel = channel
+        self.tasks_per_node = tasks_per_node
+        self.nodes_per_block = nodes_per_block
+        self.init_blocks = init_blocks
+        self.min_blocks = min_blocks
+        self.max_blocks = max_blocks
+        self.parallelism = parallelism
+        self.overrides = overrides
+        self.secret = secret
+        self.user_id = user_id
+        self.group_id = group_id
+        self.run_as_non_root = run_as_non_root
 
-        self.secret = None
-        if 'secret' in self.config['execution']:
-            self.secret = self.config['execution']['secret']
+        self.kube_client = client.ExtensionsV1beta1Api()
 
         # Dictionary that keeps track of jobs, keyed on job_id
         self.resources = {}
 
     def submit(self, cmd_string, blocksize, job_name="parsl.auto"):
         """ Submit a job
-
         Args:
              - cmd_string  :(String) - Name of the container to initiate
              - blocksize   :(float) - Number of replicas
-
         Kwargs:
              - job_name (String): Name for job, must be unique
-
         Returns:
              - None: At capacity, cannot provision more
              - job_id: (string) Identifier for the job
-
         """
         if not self.resources:
             job_name = "{0}-{1}".format(job_name, time.time()).split(".")[0]
@@ -89,7 +110,7 @@ def submit(self, cmd_string, blocksize, job_name="parsl.auto"):
                                                              str(time.time()).split('.')[0])
 
             formatted_cmd = template_string.format(command=cmd_string,
-                                                   overrides=self.config["execution"]["block"]["options"].get("overrides", ''))
+                                                   overrides=self.overrides)
 
             print("Creating replicas :", self.init_blocks)
             self.deployment_obj = self._create_deployment_object(job_name,
@@ -107,17 +128,13 @@ def submit(self, cmd_string, blocksize, job_name="parsl.auto"):
     def status(self, job_ids):
         """ Get the status of a list of jobs identified by the job identifiers
         returned from the submit request.
-
         Args:
              - job_ids (list) : A list of job identifiers
-
         Returns:
              - A list of status from ['PENDING', 'RUNNING', 'CANCELLED', 'COMPLETED',
                'FAILED', 'TIMEOUT'] corresponding to each job_id in the job_ids list.
-
         Raises:
              - ExecutionProviderExceptions or its subclasses
-
         """
         self._status()
         # This is a hack
@@ -125,10 +142,8 @@ def status(self, job_ids):
 
     def cancel(self, job_ids):
         """ Cancels the jobs specified by a list of job ids
-
         Args:
         job_ids : [<job_id> ...]
-
         Returns :
         [True/False...] : If the cancel operation fails the entire list will be False.
         """
@@ -144,10 +159,8 @@ def cancel(self, job_ids):
 
     def _status(self):
         """ Internal: Do not call. Returns the status list for a list of job_ids
-
         Args:
               self
-
         Returns:
               [status...] : Status list of all jobs
         """
@@ -164,15 +177,12 @@ def _create_deployment_object(self, job_name, job_image,
                                   engine_json_file='~/.ipython/profile_default/security/ipcontroller-engine.json',
                                   engine_dir='.'):
         """ Create a kubernetes deployment for the job.
-
         Args:
               - job_name (string) : Name of the job and deployment
               - job_image (string) : Docker image to launch
-
         KWargs:
              - port (integer) : Container port
              - replicas : Number of replica containers to maintain
-
         Returns:
               - True: The deployment object to launch
         """
@@ -180,13 +190,11 @@ def _create_deployment_object(self, job_name, job_image,
         # sorry, quick hack that doesn't pass this stuff through to test it works.
         # TODO it also doesn't only add what is set :(
         security_context = None
-        if 'security' in self.config['execution']:
+        if self.user_id and self.group_id:
             security_context = client.V1SecurityContext(run_as_group=self.group_id,
                                                         run_as_user=self.user_id,
                                                         run_as_non_root=self.run_as_non_root)
-            #                    self.user_id = None
-            #                    self.group_id = None
-            #                    self.run_as_non_root = None
+
         # Create the enviornment variables and command to initiate IPP
         environment_vars = client.V1EnvVar(name="TEST", value="SOME DATA")
 


Test patch: