Repository: DataBiosphere/toil

Issue:
Kubernetes memory limits ought to be greater then requests
The Kubernetes way of doing things is to give a bit of slack to processes and set limits that are greater than the requests, to avoid killing jobs unnecessarily, while making efficient use of resources, by allowing a slight overcommit. We should maybe do that when we issue jobs.

Also, I have a run I want to resume where my memory limits are too low and the only way I can get my jobs to run is if Kubernetes gives them a bit more memory.

┆Issue is synchronized with this [Jira Task](https://ucsc-cgl.atlassian.net/browse/TOIL-459)
┆Issue Number: TOIL-459



Discussion:

Resolving patch:
diff --git a/src/toil/batchSystems/kubernetes.py b/src/toil/batchSystems/kubernetes.py
--- a/src/toil/batchSystems/kubernetes.py
+++ b/src/toil/batchSystems/kubernetes.py
@@ -186,7 +186,10 @@ def issueBatchJob(self, jobNode):
             requirements_dict = {'cpu': jobNode.cores,
                                  'memory': max(jobNode.memory, 1024 * 1024 * 512),
                                  'ephemeral-storage': max(jobNode.disk, 1024 * 1024 * 512)}
-            resources = kubernetes.client.V1ResourceRequirements(limits=requirements_dict,
+            # Set a higher limit to give jobs some room to go over what they
+            # think they need, as is the Kubernetes way.
+            limits_dict = {k: int(v * 1.5) for k, v in requirements_dict.items()}
+            resources = kubernetes.client.V1ResourceRequirements(limits=limits_dict,
                                                                  requests=requirements_dict)
             
             # Collect volumes and mounts
@@ -408,18 +411,16 @@ def getUpdatedBatchJob(self, maxWait):
 
             if self.enableWatching:
                 for j in self._ourJobObjects():
-                    logger.debug(j.spec.template.metadata.labels[u'job-name'], type(j.spec.template.metadata.labels[u'job-name']))
                     for event in w.stream(self.coreApi.list_namespaced_pod, self.namespace, timeout_seconds=maxWait):
                         pod = event['object']
                         if pod.metadata.name.startswith(self.jobPrefix):
-                            logger.info("Event: %s %s %s" % (event['type'],event['object'].kind, event['object'].metadata.name))
                             if pod.status.phase == 'Failed' or pod.status.phase == 'Succeeded':
                                 containerStatuses =  pod.status.container_statuses
-                                logger.info("FINISHED")
+                                logger.debug("FINISHED")
                                 if containerStatuses is None or len(containerStatuses) == 0: 
                                     logger.debug("No job container statuses for job %s" % (pod.metadata.owner_references[0].name))
                                     return (int(pod.metadata.owner_references[0].name[len(self.jobPrefix):]), -1, 0)
-                                logger.info("REASON: %s Eixt Code: %s" % (pod.status.container_statuses[0].state.terminated.reason,
+                                logger.info("REASON: %s Exit Code: %s" % (pod.status.container_statuses[0].state.terminated.reason,
                                     pod.status.container_statuses[0].state.terminated.exit_code))
                                 jobID = int(pod.metadata.owner_references[0].name[len(self.jobPrefix):])
                                 terminated = pod.status.container_statuses[0].state.terminated


Test patch: