Repository: DataBiosphere/toil

Issue:
Please consider to add retry limit for SLURM
When using SLURM **sbatch** in busy cluster, sometime you will see error like this: 
```
sbatch:error:Batch job submission failed: Socket timed out on send/recv operation
```
This will happen for command **sacct**, **squeue** and **scontrol** as well.

When we read the code, we noticed that "src/toil/batchSystems/slurm.py" has caught the exception, but there is no code to retry submiting, it just print out some error messages. This will cause the workflow fail:
```
def submitJob(self, subLine):
    try:
        output = subprocess.check_output(subLine, stderr=subprocess.STDOUT)
        # sbatch prints a line like 'Submitted batch job 2954103'
        result = int(output.strip().split()[-1])
        logger.debug("sbatch submitted job %d", result)
        return result
    except subprocess.CalledProcessError as e:
        logger.error("sbatch command failed with code %d: %s", e.returncode, e.output)
        raise e
    except OSError as e:
        logger.error("sbatch command failed")
        raise e
```

Would you please consider to implement the retry function, and let user choose how many times to retry submitting a job? 
Note, there might be a same issue for SGE and Torque.

┆Issue is synchronized with this [JIRA Story](https://ucsc-cgl.atlassian.net/browse/TOIL-349)
┆Issue Number: TOIL-349



Discussion:

Resolving patch:
diff --git a/src/toil/batchSystems/abstractGridEngineBatchSystem.py b/src/toil/batchSystems/abstractGridEngineBatchSystem.py
--- a/src/toil/batchSystems/abstractGridEngineBatchSystem.py
+++ b/src/toil/batchSystems/abstractGridEngineBatchSystem.py
@@ -25,6 +25,7 @@
 from six.moves.queue import Empty, Queue
 from future.utils import with_metaclass
 
+from toil import subprocess
 from toil.lib.objects import abstractclassmethod
 
 from toil.batchSystems.abstractBatchSystem import BatchSystemLocalSupport
@@ -32,6 +33,21 @@
 logger = logging.getLogger(__name__)
 
 
+def with_retries(operation, *args, **kwargs):
+    retries = 3
+    latest_err = None
+    while retries:
+        retries -= 1
+        try:
+            return operation(*args, **kwargs)
+        except subprocess.CalledProcessError as err:
+            latest_err = err
+            logger.error(
+                "Operation %s failed with code %d: %s",
+                operation, err.returncode, err.output)
+    raise latest_err
+
+
 class AbstractGridEngineBatchSystem(BatchSystemLocalSupport):
     """
     A partial implementation of BatchSystemSupport for batch systems run on a
@@ -41,7 +57,8 @@ class AbstractGridEngineBatchSystem(BatchSystemLocalSupport):
 
     class Worker(with_metaclass(ABCMeta, Thread)):
 
-        def __init__(self, newJobsQueue, updatedJobsQueue, killQueue, killedJobsQueue, boss):
+        def __init__(self, newJobsQueue, updatedJobsQueue, killQueue,
+                     killedJobsQueue, boss):
             """
             Abstract worker interface class. All instances are created with five
             initial arguments (below). Note the Queue instances passed are empty.
@@ -50,12 +67,14 @@ def __init__(self, newJobsQueue, updatedJobsQueue, killQueue, killedJobsQueue, b
             :param updatedJobsQueue: a Queue of jobs that have been updated
             :param killQueue: a Queue of active jobs that need to be killed
             :param killedJobsQueue: Queue of killed jobs for this worker
-            :param boss: the AbstractGridEngineBatchSystem instance that controls this AbstractGridEngineWorker
+            :param boss: the AbstractGridEngineBatchSystem instance that
+                         controls this AbstractGridEngineWorker
 
             """
             Thread.__init__(self)
             self.boss = boss
-            self.boss.config.statePollingWait = self.boss.config.statePollingWait or self.boss.getWaitDuration()
+            self.boss.config.statePollingWait = \
+                self.boss.config.statePollingWait or self.boss.getWaitDuration()
             self.newJobsQueue = newJobsQueue
             self.updatedJobsQueue = updatedJobsQueue
             self.killQueue = killQueue
@@ -97,8 +116,9 @@ def forgetJob(self, jobID):
 
         def createJobs(self, newJob):
             """
-            Create a new job with the Toil job ID. Implementation-specific; called
-            by AbstractGridEngineWorker.run()
+            Create a new job with the Toil job ID.
+
+            Implementation-specific; called by AbstractGridEngineWorker.run()
 
             :param string newJob: Toil job ID
             """
@@ -107,16 +127,15 @@ def createJobs(self, newJob):
             if newJob is not None:
                 self.waitingJobs.append(newJob)
             # Launch jobs as necessary:
-            while len(self.waitingJobs) > 0 and len(self.runningJobs) < int(self.boss.config.maxLocalJobs):
+            while len(self.waitingJobs) > 0 and \
+                    len(self.runningJobs) < int(self.boss.config.maxLocalJobs):
                 activity = True
                 jobID, cpu, memory, command = self.waitingJobs.pop(0)
 
                 # prepare job submission command
                 subLine = self.prepareSubmission(cpu, memory, jobID, command)
                 logger.debug("Running %r", subLine)
-
-                # submit job and get batch system ID
-                batchJobID = self.submitJob(subLine)
+                batchJobID = with_retries(self.submitJob, subLine)
                 logger.debug("Submitted job %s", str(batchJobID))
 
                 # Store dict for mapping Toil job ID to batch job ID
@@ -165,7 +184,7 @@ def killJobs(self):
             while killList:
                 for jobID in list(killList):
                     batchJobID = self.getBatchSystemID(jobID)
-                    if self.getJobExitCode(batchJobID) is not None:
+                    if with_retries(self.getJobExitCode, batchJobID) is not None:
                         logger.debug('Adding jobID %s to killedJobsQueue', jobID)
                         self.killedJobsQueue.put(jobID)
                         killList.remove(jobID)
@@ -188,7 +207,7 @@ def checkOnJobs(self):
             activity = False
             for jobID in list(self.runningJobs):
                 batchJobID = self.getBatchSystemID(jobID)
-                status = self.getJobExitCode(batchJobID)
+                status = with_retries(self.getJobExitCode, batchJobID)
                 if status is not None:
                     activity = True
                     self.updatedJobsQueue.put((jobID, status))
@@ -276,7 +295,8 @@ def getJobExitCode(self, batchJobID):
             raise NotImplementedError()
 
     def __init__(self, config, maxCores, maxMemory, maxDisk):
-        super(AbstractGridEngineBatchSystem, self).__init__(config, maxCores, maxMemory, maxDisk)
+        super(AbstractGridEngineBatchSystem, self).__init__(
+            config, maxCores, maxMemory, maxDisk)
         self.config = config
 
         self.currentJobs = set()
@@ -291,8 +311,8 @@ def __init__(self, config, maxCores, maxMemory, maxDisk):
         self.killQueue = Queue()
         self.killedJobsQueue = Queue()
         # get the associated worker class here
-        self.worker = self.Worker(self.newJobsQueue, self.updatedJobsQueue, self.killQueue,
-                              self.killedJobsQueue, self)
+        self.worker = self.Worker(self.newJobsQueue, self.updatedJobsQueue,
+                                  self.killQueue, self.killedJobsQueue, self)
         self.worker.start()
         self._getRunningBatchJobIDsTimestamp = None
         self._getRunningBatchJobIDsCache = {}
@@ -346,16 +366,19 @@ def getIssuedBatchJobIDs(self):
         return list(self.getIssuedLocalJobIDs()) + list(self.currentJobs)
 
     def getRunningBatchJobIDs(self):
-        """Retrieve running job IDs from local and batch scheduler.
+        """
+        Retrieve running job IDs from local and batch scheduler.
 
         Respects statePollingWait and will return cached results if not within
         time period to talk with the scheduler.
         """
-        if (self._getRunningBatchJobIDsTimestamp and
-             (datetime.now() - self._getRunningBatchJobIDsTimestamp).total_seconds() < self.config.statePollingWait):
+        if (self._getRunningBatchJobIDsTimestamp and (
+                datetime.now() -
+                self._getRunningBatchJobIDsTimestamp).total_seconds() <
+                self.config.statePollingWait):
             batchIds = self._getRunningBatchJobIDsCache
         else:
-            batchIds = self.worker.getRunningJobIDs()
+            batchIds = with_retries(self.worker.getRunningJobIDs)
             self._getRunningBatchJobIDsCache = batchIds
             self._getRunningBatchJobIDsTimestamp = datetime.now()
         batchIds.update(self.getRunningLocalJobIDs())
@@ -389,7 +412,7 @@ def shutdown(self):
     def setEnv(self, name, value=None):
         if value and ',' in value:
             raise ValueError(type(self).__name__ + " does not support commata in environment variable values")
-        return super(AbstractGridEngineBatchSystem,self).setEnv(name, value)
+        return super(AbstractGridEngineBatchSystem, self).setEnv(name, value)
 
     @classmethod
     def getWaitDuration(self):
diff --git a/src/toil/batchSystems/slurm.py b/src/toil/batchSystems/slurm.py
--- a/src/toil/batchSystems/slurm.py
+++ b/src/toil/batchSystems/slurm.py
@@ -71,9 +71,6 @@ def submitJob(self, subLine):
                 result = int(output.strip().split()[-1])
                 logger.debug("sbatch submitted job %d", result)
                 return result
-            except subprocess.CalledProcessError as e:
-                logger.error("sbatch command failed with code %d: %s", e.returncode, e.output)
-                raise e
             except OSError as e:
                 logger.error("sbatch command failed")
                 raise e


Test patch: