Repository: DataBiosphere/toil

Issue:
Kubernetes batch system can't pick up config from a pod's service account
The config loading method we're using to connect to Kubernetes, as described in the module's README, only works when your credentials come from a config file. When they come from a pod's service account, `kubectl` finds them automatically, but the `kubernetes` module needs you to manually call a differnet utility method (`config.load_incluster_config()`) instead.

See:

* Example: https://github.com/kubernetes-client/python/blob/master/examples/in_cluster_config.py

* Implementation/utility method docs: https://github.com/kubernetes-client/python-base/blob/4c1ab55553fa7d0479bd5b270c2b303cb056dd4b/config/incluster_config.py

* Kubernetes issue: https://github.com/kubernetes-client/python/issues/1005

┆Issue is synchronized with this [Jira Task](https://ucsc-cgl.atlassian.net/browse/TOIL-449)
┆Issue Number: TOIL-449



Discussion:

Resolving patch:
diff --git a/src/toil/batchSystems/kubernetes.py b/src/toil/batchSystems/kubernetes.py
--- a/src/toil/batchSystems/kubernetes.py
+++ b/src/toil/batchSystems/kubernetes.py
@@ -73,19 +73,32 @@ def __init__(self, config, maxCores, maxMemory, maxDisk):
         logging.getLogger('requests_oauthlib').setLevel(logging.ERROR)
 
         try:
-            # Load ~/.kube/config
+            # Load ~/.kube/config or KUBECONFIG
             kubernetes.config.load_kube_config()
+            
+            # We loaded it; we need to figure out our namespace the config-file way
+            
+            # Find all contexts and the active context.
+            # The active context gets us our namespace.
+            contexts, activeContext = kubernetes.config.list_kube_config_contexts()
+            if not contexts:
+                raise RuntimeError("No Kubernetes contexts available in ~/.kube/config or $KUBECONFIG")
+                
+            # Identify the namespace to work in
+            self.namespace = activeContext.get('context', {}).get('namespace', 'default')
+            
         except TypeError:
-            raise RuntimeError('Could not load Kubernetes configuration. Does ~/.kube/config or $KUBECONFIG exist?')
+            # Didn't work. Try pod-based credentials in case we are in a pod.
+            try:
+                kubernetes.config.load_incluster_config()
+                
+                # We got pod-based credentials. Our namespace comes from a particular file.
+                self.namespace = open("/var/run/secrets/kubernetes.io/serviceaccount/namespace", 'r').read().strip()
+                
+            except kubernetes.config.ConfigException:
+                raise RuntimeError('Could not load Kubernetes configuration from ~/.kube/config, $KUBECONFIG, or current pod.')
 
-        # Find all contexts and the active context.
-        # The active context gets us our namespace.
-        contexts, activeContext = kubernetes.config.list_kube_config_contexts()
-        if not contexts:
-            raise RuntimeError("No Kubernetes contexts available in ~/.kube/config or $KUBECONFIG")
-            
-        # Identify the namespace to work in
-        self.namespace = activeContext.get('context', {}).get('namespace', 'default')
+        
 
         # Make a Kubernetes-acceptable version of our username: not too long,
         # and all lowercase letters, numbers, or - or .
@@ -483,8 +496,18 @@ def _getUpdatedBatchJobImmediately(self):
                 if pod is None:
                     # Skip jobs with no pod
                     continue
+                    
+                # Get the statuses of the pod's containers
+                containerStatuses = pod.status.container_statuses
+                if containerStatuses is None or len(containerStatuses) == 0:
+                    # Pod exists but has no container statuses
+                    # This happens when the pod is just "Scheduled"
+                    # ("PodScheduled" status event) and isn't actually starting
+                    # to run yet.
+                    # Can't be stuck in ImagePullBackOff
+                    continue
 
-                waitingInfo = getattr(pod.status.container_statuses[0].state, 'waiting')
+                waitingInfo = getattr(getattr(pod.status.container_statuses[0], 'state', None), 'waiting', None)
                 if waitingInfo is not None and waitingInfo.reason == 'ImagePullBackOff':
                     # Assume it will never finish, even if the registry comes back or whatever.
                     # We can get into this state when we send in a non-existent image.
@@ -513,30 +536,43 @@ def _getUpdatedBatchJobImmediately(self):
             if chosenFor == 'done' or chosenFor == 'failed':
                 # The job actually finished or failed
 
-                # Get the termination info from the pod
-                logger.debug(pod.status.container_statuses)
-                terminatedInfo = getattr(pod.status.container_statuses[0].state, 'terminated')
-                if terminatedInfo is not None:
-                    # Extract the exit code
-                    exitCode = terminatedInfo.exit_code
-
-                    # Compute how long the job ran for (subtract datetimes)
-                    # We need to look at the pod's start time because the job's
-                    # start time is just when the job is created.
-                    # And we need to look at the pod's end time because the
-                    # job only gets a completion time if successful.
-                    runtime = (terminatedInfo.finished_at - 
-                               pod.status.start_time).total_seconds()
-
-                    if chosenFor == 'failed':
-                        # Warn the user with the failed pod's log
-                        # TODO: cut this down somehow?
-                        logger.warning('Log from failed pod: %s', self._getLogForPod(pod))
-                else:
-                    logger.warning('Exit code and runtime unavailable; pod stopped without container terminating')
+                # Get the statuses of the pod's containers
+                containerStatuses = pod.status.container_statuses
+                
+                if containerStatuses is None or len(containerStatuses) == 0:
+                    # No statuses available.
+                    # This happens when a pod is "Scheduled". But how could a
+                    # 'done' or 'failed' pod be merely "Scheduled"?
+                    # Complain so we can find out.
+                    logger.warning('Exit code and runtime unavailable; pod has no container statuses')
+                    logger.warning('Pod: %s', str(pod))
                     exitCode = -1
                     runtime = 0
-                    
+                else:
+                    # Get the termination info from the pod's main (only) container
+                    terminatedInfo = getattr(getattr(containerStatuses[0], 'state', None), 'terminated', None)
+                    if terminatedInfo is None:
+                        logger.warning('Exit code and runtime unavailable; pod stopped without container terminating')
+                        logger.warning('Pod: %s', str(pod))
+                        exitCode = -1
+                        runtime = 0
+                    else:
+                        # Extract the exit code
+                        exitCode = terminatedInfo.exit_code
+
+                        # Compute how long the job ran for (subtract datetimes)
+                        # We need to look at the pod's start time because the job's
+                        # start time is just when the job is created.
+                        # And we need to look at the pod's end time because the
+                        # job only gets a completion time if successful.
+                        runtime = (terminatedInfo.finished_at - 
+                                   pod.status.start_time).total_seconds()
+
+                        if chosenFor == 'failed':
+                            # Warn the user with the failed pod's log
+                            # TODO: cut this down somehow?
+                            logger.warning('Log from failed pod: %s', self._getLogForPod(pod))
+            
             else:
                 # The job has gotten stuck
 
@@ -717,7 +753,7 @@ def executor():
     logger.debug("Starting executor")
 
     if len(sys.argv) != 2:
-        log.error('Executor requires exactly one base64-encoded argument')
+        logger.error('Executor requires exactly one base64-encoded argument')
         sys.exit(1)
 
     # Take in a base64-encoded pickled dict as our first argument and decode it


Test patch: