Repository: DataBiosphere/toil

Issue:
Rare exception in Mesos scheduler
```
Failed to call scheduler's statusUpdate
Traceback (most recent call last):
  File "/usr/local/lib/python2.7/dist-packages/toil/batchSystems/mesos/batchSystem.py", line 458, in statusUpdate
    exitStatus = int(update.message)
ValueError: invalid literal for int() with base 10: ''
```

Rare exception in Mesos scheduler
```
Failed to call scheduler's statusUpdate
Traceback (most recent call last):
  File "/usr/local/lib/python2.7/dist-packages/toil/batchSystems/mesos/batchSystem.py", line 458, in statusUpdate
    exitStatus = int(update.message)
ValueError: invalid literal for int() with base 10: ''
```



Discussion:

Resolving patch:
diff --git a/src/toil/batchSystems/mesos/batchSystem.py b/src/toil/batchSystems/mesos/batchSystem.py
--- a/src/toil/batchSystems/mesos/batchSystem.py
+++ b/src/toil/batchSystems/mesos/batchSystem.py
@@ -455,8 +455,13 @@ def statusUpdate(self, driver, update):
         if update.state == mesos_pb2.TASK_FINISHED:
             self.__updateState(taskID, 0)
         elif update.state == mesos_pb2.TASK_FAILED:
-            exitStatus = int(update.message)
-            log.warning('Task %i failed with exit status %i', taskID, exitStatus)
+            try:
+                exitStatus = int(update.message)
+            except ValueError:
+                exitStatus = 255
+                log.warning("Task %i failed with message '%s'", taskID, update.message)
+            else:
+                log.warning('Task %i failed with exit status %i', taskID, exitStatus)
             self.__updateState(taskID, exitStatus)
         elif update.state in (mesos_pb2.TASK_LOST, mesos_pb2.TASK_KILLED, mesos_pb2.TASK_ERROR):
             log.warning("Task %i is in unexpected state %s with message '%s'",
diff --git a/src/toil/batchSystems/mesos/executor.py b/src/toil/batchSystems/mesos/executor.py
--- a/src/toil/batchSystems/mesos/executor.py
+++ b/src/toil/batchSystems/mesos/executor.py
@@ -98,21 +98,25 @@ def launchTask(self, driver, task):
         def runTask():
             log.debug("Running task %s", task.task_id.value)
             sendUpdate(mesos_pb2.TASK_RUNNING)
-            popen = runJob(pickle.loads(task.data))
-            self.runningTasks[task.task_id.value] = popen.pid
             try:
-                exitStatus = popen.wait()
-                if 0 == exitStatus:
-                    sendUpdate(mesos_pb2.TASK_FINISHED)
-                elif -9 == exitStatus:
-                    sendUpdate(mesos_pb2.TASK_KILLED)
-                else:
-                    sendUpdate(mesos_pb2.TASK_FAILED, message=str(exitStatus))
+                popen = runJob(pickle.loads(task.data))
+                self.runningTasks[task.task_id.value] = popen.pid
+                try:
+                    exitStatus = popen.wait()
+                    if 0 == exitStatus:
+                        sendUpdate(mesos_pb2.TASK_FINISHED)
+                    elif -9 == exitStatus:
+                        sendUpdate(mesos_pb2.TASK_KILLED)
+                    else:
+                        sendUpdate(mesos_pb2.TASK_FAILED, message=str(exitStatus))
+                finally:
+                    del self.runningTasks[task.task_id.value]
             except:
-                exc_type, exc_value, exc_trace = sys.exc_info()
-                sendUpdate(mesos_pb2.TASK_FAILED, message=str(traceback.format_exception_only(exc_type, exc_value)))
-            finally:
-                del self.runningTasks[task.task_id.value]
+                exc_info = sys.exc_info()
+                log.error('Exception while running task:', exc_info=exc_info)
+                exc_type, exc_value, exc_trace = exc_info
+                sendUpdate(mesos_pb2.TASK_FAILED,
+                           message=''.join(traceback.format_exception_only(exc_type, exc_value)))
 
         def runJob(job):
             """


Test patch: