Repository: spotify/luigi

Issue:
[contrib.kubernetes] Hangs on 'Pod errors: Unschedulable'
When asking for more resources (e.g. CPU) than the Kubernetes cluster can allocate, Kubernetes fails the job (rightfully) with a status `Pod errors: Unschedulable`.

The `KubernetesJobTask` logic, however, fails to catch this error and hangs:
```
DEBUG: Kubernetes job example-luigi-b8ffbfbbeb7b429bb8cc1e81c6a879c9 is still running
DEBUG: Kubernetes job example-luigi-b8ffbfbbeb7b429bb8cc1e81c6a879c9 is still running
DEBUG: Kubernetes job example-luigi-b8ffbfbbeb7b429bb8cc1e81c6a879c9 is still running
...
```
(FWIW, I'm using GKE here)


Discussion:
@mcapuccini ^
Hi @yonromai. Thanks for reporting the issue. The problem should on this block of code: https://github.com/spotify/luigi/blob/6ce970846c0ca8c2850bc168b686340e15982b2b/luigi/contrib/kubernetes.py#L160-L171

It only checks for pod status "running", "succeeded" and throws error for everything else. I think that `pykube`, the library that we are currently using for the Kubernetes Task, returns running when the pod is unschedulable, and that we might need to run an additional command to query the unschedulable status. Unfortunately I don't have time to work on it at the moment, but since you are getting the error in one of your settings, it would be a nice contribution if you could sort it out. I'd be happy to review your patch 🙂 .
@mcapuccini thanks for coming back to me. 

I took a look at it yesterday, I think the problem might rather be in the `self.__get_job_status()`. 
I played with it and printed the contents of `pod.obj['spec']`, `node.obj['spec']` and `job.obj`. All of these objects had the same data whether the job was actually scheduled or not (because too much resources was requested). 
Also,  `node.unschedulable` was `False` in both cases.

I'm wondering if the problem could be with pykube/GKE compatibility. 

Resolving patch:
diff --git a/luigi/contrib/kubernetes.py b/luigi/contrib/kubernetes.py
--- a/luigi/contrib/kubernetes.py
+++ b/luigi/contrib/kubernetes.py
@@ -32,18 +32,19 @@
 
 Written and maintained by Marco Capuccini (@mcapuccini).
 """
-
-import luigi
 import logging
-import uuid
 import time
+import uuid
+from datetime import datetime
+
+import luigi
 
 logger = logging.getLogger('luigi-interface')
 
 try:
     from pykube.config import KubeConfig
     from pykube.http import HTTPClient
-    from pykube.objects import Job
+    from pykube.objects import Job, Pod
 except ImportError:
     logger.warning('pykube is not installed. KubernetesJobTask requires pykube.')
 
@@ -61,21 +62,21 @@ class kubernetes(luigi.Config):
 
 
 class KubernetesJobTask(luigi.Task):
-
     __POLL_TIME = 5  # see __track_job
-    kubernetes_config = kubernetes()
+    _kubernetes_config = None  # Needs to be loaded at runtime
 
     def _init_kubernetes(self):
         self.__logger = logger
         self.__logger.debug("Kubernetes auth method: " + self.auth_method)
-        if(self.auth_method == "kubeconfig"):
+        if self.auth_method == "kubeconfig":
             self.__kube_api = HTTPClient(KubeConfig.from_file(self.kubeconfig_path))
-        elif(self.auth_method == "service-account"):
+        elif self.auth_method == "service-account":
             self.__kube_api = HTTPClient(KubeConfig.from_service_account())
         else:
             raise ValueError("Illegal auth_method")
         self.job_uuid = str(uuid.uuid4().hex)
-        self.uu_name = self.name + "-luigi-" + self.job_uuid
+        now = datetime.utcnow()
+        self.uu_name = "%s-%s-%s" % (self.name, now.strftime('%Y%m%d%H%M%S'), self.job_uuid[:16])
 
     @property
     def auth_method(self):
@@ -157,18 +158,52 @@ def max_retrials(self):
         """
         return self.kubernetes_config.max_retrials
 
+    @property
+    def delete_on_success(self):
+        """
+        Delete the Kubernetes workload if the job has ended successfully.
+        """
+        return True
+
+    @property
+    def print_pod_logs_on_exit(self):
+        """
+        Fetch and print the pod logs once the job is completed.
+        """
+        return False
+
+    @property
+    def active_deadline_seconds(self):
+        """
+        Time allowed to successfully schedule pods.
+        See: https://kubernetes.io/docs/concepts/workloads/controllers/jobs-run-to-completion/#job-termination-and-cleanup
+        """
+        return 100
+
+    @property
+    def kubernetes_config(self):
+        if not self._kubernetes_config:
+            self._kubernetes_config = kubernetes()
+        return self._kubernetes_config
+
     def __track_job(self):
         """Poll job status while active"""
-        while (self.__get_job_status() == "running"):
-            self.__logger.debug("Kubernetes job " + self.uu_name
-                                + " is still running")
+        while not self.__verify_job_has_started():
             time.sleep(self.__POLL_TIME)
-        if(self.__get_job_status() == "succeeded"):
-            self.__logger.info("Kubernetes job " + self.uu_name + " succeeded")
-            # Use signal_complete to notify of job completion
-            self.signal_complete()
-        else:
-            raise RuntimeError("Kubernetes job " + self.uu_name + " failed")
+            self.__logger.debug("Waiting for Kubernetes job " + self.uu_name + " to start")
+        self.__print_kubectl_hints()
+
+        status = self.__get_job_status()
+        while status == "RUNNING":
+            self.__logger.debug("Kubernetes job " + self.uu_name + " is running")
+            time.sleep(self.__POLL_TIME)
+            status = self.__get_job_status()
+
+        assert status != "FAILED", "Kubernetes job " + self.uu_name + " failed"
+
+        # status == "SUCCEEDED"
+        self.__logger.info("Kubernetes job " + self.uu_name + " succeeded")
+        self.signal_complete()
 
     def signal_complete(self):
         """Signal job completion for scheduler and dependent tasks.
@@ -181,27 +216,96 @@ def signal_complete(self):
         """
         pass
 
+    def __get_pods(self):
+        pod_objs = Pod.objects(self.__kube_api) \
+            .filter(selector="job-name=" + self.uu_name) \
+            .response['items']
+        return [Pod(self.__kube_api, p) for p in pod_objs]
+
+    def __get_job(self):
+        jobs = Job.objects(self.__kube_api) \
+            .filter(selector="luigi_task_id=" + self.job_uuid) \
+            .response['items']
+        assert len(jobs) == 1, "Kubernetes job " + self.uu_name + " not found"
+        return Job(self.__kube_api, jobs[0])
+
+    def __print_pod_logs(self):
+        for pod in self.__get_pods():
+            logs = pod.logs(timestamps=True).strip()
+            self.__logger.info("Fetching logs from " + pod.name)
+            if len(logs) > 0:
+                for l in logs.split('\n'):
+                    self.__logger.info(l)
+
+    def __print_kubectl_hints(self):
+        self.__logger.info("To stream Pod logs, use:")
+        for pod in self.__get_pods():
+            self.__logger.info("`kubectl logs -f pod/%s`" % pod.name)
+
+    def __verify_job_has_started(self):
+        """Asserts that the job has successfully started"""
+        # Verify that the job started
+        self.__get_job()
+
+        # Verify that the pod started
+        pods = self.__get_pods()
+
+        assert len(pods) > 0, "No pod scheduled by " + self.uu_name
+        for pod in pods:
+            status = pod.obj['status']
+            for cont_stats in status.get('containerStatuses', []):
+                if 'terminated' in cont_stats['state']:
+                    t = cont_stats['state']['terminated']
+                    err_msg = "Pod %s %s (exit code %d). Logs: `kubectl logs pod/%s`" % (
+                        pod.name, t['reason'], t['exitCode'], pod.name)
+                    assert t['exitCode'] == 0, err_msg
+
+                if 'waiting' in cont_stats['state']:
+                    wr = cont_stats['state']['waiting']['reason']
+                    assert wr == 'ContainerCreating', "Pod %s %s. Logs: `kubectl logs pod/%s`" % (
+                        pod.name, wr, pod.name)
+
+            for cond in status['conditions']:
+                if 'message' in cond:
+                    if cond['reason'] == 'ContainersNotReady':
+                        return False
+                    assert cond['status'] != 'False', \
+                        "[ERROR] %s - %s" % (cond['reason'], cond['message'])
+        return True
+
     def __get_job_status(self):
         """Return the Kubernetes job status"""
-        # Look for the required job
-        jobs = Job.objects(self.__kube_api).filter(selector="luigi_task_id="
-                                                            + self.job_uuid)
-        # Raise an exception if no such job found
-        if len(jobs.response["items"]) == 0:
-            raise RuntimeError("Kubernetes job " + self.uu_name + " not found")
         # Figure out status and return it
-        job = Job(self.__kube_api, jobs.response["items"][0])
-        if ("succeeded" in job.obj["status"] and job.obj["status"]["succeeded"] > 0):
-            job.scale(replicas=0)  # Downscale the job, but keep it for logging
-            return "succeeded"
-        if ("failed" in job.obj["status"]):
+        job = self.__get_job()
+
+        if "succeeded" in job.obj["status"] and job.obj["status"]["succeeded"] > 0:
+            job.scale(replicas=0)
+            if self.print_pod_logs_on_exit:
+                self.__print_pod_logs()
+            if self.delete_on_success:
+                self.__delete_job_cascade(job)
+            return "SUCCEEDED"
+
+        if "failed" in job.obj["status"]:
             failed_cnt = job.obj["status"]["failed"]
             self.__logger.debug("Kubernetes job " + self.uu_name
                                 + " status.failed: " + str(failed_cnt))
-            if (failed_cnt > self.max_retrials):
+            if self.print_pod_logs_on_exit:
+                self.__print_pod_logs()
+            if failed_cnt > self.max_retrials:
                 job.scale(replicas=0)  # avoid more retrials
-                return "failed"
-        return "running"
+                return "FAILED"
+        return "RUNNING"
+
+    def __delete_job_cascade(self, job):
+        delete_options_cascade = {
+            "kind": "DeleteOptions",
+            "apiVersion": "v1",
+            "propagationPolicy": "Background"
+        }
+        r = self.__kube_api.delete(json=delete_options_cascade, **job.api_kwargs())
+        if r.status_code != 200:
+            self.__kube_api.raise_for_status(r)
 
     def run(self):
         self._init_kubernetes()
@@ -217,6 +321,7 @@ def run(self):
                 }
             },
             "spec": {
+                "activeDeadlineSeconds": self.active_deadline_seconds,
                 "template": {
                     "metadata": {
                         "name": self.uu_name
@@ -228,7 +333,7 @@ def run(self):
         # Update user labels
         job_json['metadata']['labels'].update(self.labels)
         # Add default restartPolicy if not specified
-        if ("restartPolicy" not in self.spec_schema):
+        if "restartPolicy" not in self.spec_schema:
             job_json["spec"]["template"]["spec"]["restartPolicy"] = "Never"
         # Submit job
         self.__logger.info("Submitting Kubernetes Job: " + self.uu_name)


Test patch: