Repository: DataBiosphere/toil

Issue:
Another Mesos scheduler failure
```
Failed to call scheduler's statusUpdate
Traceback (most recent call last):
  File "/usr/local/lib/python2.7/dist-packages/toil/batchSystems/mesos/batchSystem.py", line 456, in statusUpdate
    self.__updateState(taskID, 0)
  File "/usr/local/lib/python2.7/dist-packages/toil/batchSystems/mesos/batchSystem.py", line 433, in __updateState
    del self.runningJobMap[intID]
KeyError: 42199
```



Discussion:

Resolving patch:
diff --git a/src/toil/batchSystems/mesos/batchSystem.py b/src/toil/batchSystems/mesos/batchSystem.py
--- a/src/toil/batchSystems/mesos/batchSystem.py
+++ b/src/toil/batchSystems/mesos/batchSystem.py
@@ -433,7 +433,11 @@ def _createTask(self, jt_job, offer):
 
     def __updateState(self, intID, exitStatus):
         self.updatedJobsQueue.put((intID, exitStatus))
-        del self.runningJobMap[intID]
+        try:
+            del self.runningJobMap[intID]
+        except KeyError:
+            log.warning('Cannot find %i among running jobs. '
+                        'Sent update about its exit code of %i anyways.', intID, exitStatus)
 
     def statusUpdate(self, driver, update):
         """
@@ -458,8 +462,13 @@ def statusUpdate(self, driver, update):
         if update.state == mesos_pb2.TASK_FINISHED:
             self.__updateState(taskID, 0)
         elif update.state == mesos_pb2.TASK_FAILED:
-            exitStatus = int(update.message)
-            log.warning('Task %i failed with exit status %i', taskID, exitStatus)
+            try:
+                exitStatus = int(update.message)
+            except ValueError:
+                exitStatus = 255
+                log.warning("Task %i failed with message '%s'", taskID, update.message)
+            else:
+                log.warning('Task %i failed with exit status %i', taskID, exitStatus)
             self.__updateState(taskID, exitStatus)
         elif update.state in (mesos_pb2.TASK_LOST, mesos_pb2.TASK_KILLED, mesos_pb2.TASK_ERROR):
             log.warning("Task %i is in unexpected state %s with message '%s'",
diff --git a/src/toil/batchSystems/mesos/executor.py b/src/toil/batchSystems/mesos/executor.py
--- a/src/toil/batchSystems/mesos/executor.py
+++ b/src/toil/batchSystems/mesos/executor.py
@@ -108,22 +108,25 @@ def runTask():
                 assert self.workerCleanupInfo == taskData.workerCleanupInfo
             else:
                 self.workerCleanupInfo = taskData.workerCleanupInfo
-            popen = runJob(taskData)
-            self.runningTasks[task.task_id.value] = popen.pid
             try:
-                exitStatus = popen.wait()
-                if 0 == exitStatus:
-                    sendUpdate(mesos_pb2.TASK_FINISHED)
-                elif -9 == exitStatus:
-                    sendUpdate(mesos_pb2.TASK_KILLED)
-                else:
-                    sendUpdate(mesos_pb2.TASK_FAILED, message=str(exitStatus))
+                popen = runJob(taskData)
+                self.runningTasks[task.task_id.value] = popen.pid
+                try:
+                    exitStatus = popen.wait()
+                    if 0 == exitStatus:
+                        sendUpdate(mesos_pb2.TASK_FINISHED)
+                    elif -9 == exitStatus:
+                        sendUpdate(mesos_pb2.TASK_KILLED)
+                    else:
+                        sendUpdate(mesos_pb2.TASK_FAILED, message=str(exitStatus))
+                finally:
+                    del self.runningTasks[task.task_id.value]
             except:
-                exc_type, exc_value, exc_trace = sys.exc_info()
+                exc_info = sys.exc_info()
+                log.error('Exception while running task:', exc_info=exc_info)
+                exc_type, exc_value, exc_trace = exc_info
                 sendUpdate(mesos_pb2.TASK_FAILED,
-                           message=str(traceback.format_exception_only(exc_type, exc_value)))
-            finally:
-                del self.runningTasks[task.task_id.value]
+                           message=''.join(traceback.format_exception_only(exc_type, exc_value)))
 
         def runJob(job):
             """


Test patch: