Repository: DataBiosphere/toil

Issue:
logToMaster messages overwritten
When jobs are chained only the latest job's logToMaster messages are shown

Logging is not being setup
Log to master messages don't appear when running sortTest. We need a test for logToMaster functionality since we tend to keep breaking it.



Discussion:

+1

Regression from fb52bcedc1077077802988b43b5d3180d36c3f6b

Could this be affecting all logging and not just logToMaster? I'm not getting any exception tracebacks when my jobs break, except from the master failing because it detected a failed job.


Resolving patch:
diff --git a/src/toil/common.py b/src/toil/common.py
--- a/src/toil/common.py
+++ b/src/toil/common.py
@@ -25,7 +25,7 @@
 
 from bd2k.util.humanize import bytes2human
 
-from toil.lib.bioio import addLoggingOptions, getLogLevelString
+from toil.lib.bioio import addLoggingOptions, getLogLevelString, setLoggingFromOptions
 from toil.realtimeLogger import RealtimeLogger
 
 logger = logging.getLogger(__name__)
@@ -463,6 +463,7 @@ def __enter__(self):
         consolidate the derived configuration with the one from the previous invocation of the
         workflow.
         """
+        setLoggingFromOptions(self.options)
         self._inContextManager = True
         self.config = Config()
         self.config.setOptions(self.options)
diff --git a/src/toil/job.py b/src/toil/job.py
--- a/src/toil/job.py
+++ b/src/toil/job.py
@@ -462,9 +462,9 @@ def addToilOptions(parser):
         @staticmethod
         def startToil(job, options):
             """
-            Runs the toil workflow using the given options \
-            (see Job.Runner.getDefaultOptions and Job.Runner.addToilOptions) \
-            starting with this job.
+            Deprecated by toil.common.Toil.run. Runs the toil workflow using the given options
+            (see Job.Runner.getDefaultOptions and Job.Runner.addToilOptions) starting with this
+            job.
             :param toil.job.Job job: root job of the workflow
             :raises: toil.leader.FailedJobsException if at the end of function \
             their remain failed jobs.
diff --git a/src/toil/worker.py b/src/toil/worker.py
--- a/src/toil/worker.py
+++ b/src/toil/worker.py
@@ -187,7 +187,7 @@ def badWorker():
     workerFailed = False
     statsDict = MagicExpando()
     statsDict.jobs = []
-    messages = []
+    statsDict.workers.logsToMaster = []
     blockFn = lambda : True
     cleanCacheFn = lambda x : True
     try:
@@ -319,7 +319,6 @@ def recursiveDelete(jobWrapper2):
                                               blockFn)
                     #Get the next block function and list that will contain any messages
                     blockFn = fileStore._blockFn
-                    messages = fileStore.loggingMessages
 
                     job._execute(jobWrapper=jobWrapper,
                                            stats=statsDict if config.stats else None,
@@ -327,6 +326,9 @@ def recursiveDelete(jobWrapper2):
                                            jobStore=jobStore,
                                            fileStore=fileStore)
 
+                    # Accumulate messages from this job & any subsequent chained jobs
+                    statsDict.workers.logsToMaster += fileStore.loggingMessages
+
                     #Set the clean cache function
                     cleanCacheFn = fileStore._cleanLocalTempDir
                     
@@ -452,9 +454,6 @@ def recursiveDelete(jobWrapper2):
             statsDict.workers.clock = str(totalCPUTime - startClock)
             statsDict.workers.memory = str(totalMemoryUsage)
 
-        # logToMaster messages should be always be passed
-        statsDict.workers.logsToMaster = messages
-
         # log the worker log path here so that if the file is truncated the path can still be found
         logger.info("Worker log can be found at %s. Set --cleanWorkDir to retain this log", localTempDir)
         logger.info("Finished running the chain of jobs on this node, we ran for a total of %f seconds", time.time() - startTime)
@@ -528,7 +527,7 @@ def recursiveDelete(jobWrapper2):
             logMessages = logFile.read().splitlines()
         statsDict.logs = [Expando(jobStoreID=jobStoreID, text=logMessage) for logMessage in logMessages]
 
-    if (debugging or config.stats or messages) and not workerFailed:  # We have stats/logging to report back
+    if (debugging or config.stats or statsDict.workers.logsToMaster) and not workerFailed:  # We have stats/logging to report back
         jobStore.writeStatsAndLogging(json.dumps(statsDict))
 
     #Remove the temp dir


Test patch:
diff --git a/src/toil/test/mesos/helloWorld.py b/src/toil/test/mesos/helloWorld.py
--- a/src/toil/test/mesos/helloWorld.py
+++ b/src/toil/test/mesos/helloWorld.py
@@ -17,16 +17,24 @@
 
 from __future__ import absolute_import
 import argparse
+import os
+from toil.common import Toil
 from toil.job import Job
 
+childMessage = "The child job is now running!"
+parentMessage = "The parent job is now running!"
+
 def hello_world(job):
 
+    job.fileStore.logToMaster(parentMessage)
     with open('foo_bam.txt', 'w') as handle:
         handle.write('\nThis is a triumph...\n')
 
     # Assign FileStoreID to a given file
     foo_bam = job.fileStore.writeGlobalFile('foo_bam.txt')
 
+    os.remove('foo_bam.txt')
+
     # Spawn child
     job.addChildJobFn(hello_world_child, foo_bam, memory=100, cores=0.5, disk="3G")
 
@@ -34,7 +42,7 @@ def hello_world(job):
 def hello_world_child(job, hw):
 
     path = job.fileStore.readGlobalFile(hw)
-
+    job.fileStore.logToMaster(childMessage)
     # NOTE: path and the udpated file are stored to /tmp
     # If we want to SAVE our changes to this tmp file, we must write it out.
     with open(path, 'r') as r:
@@ -46,20 +54,22 @@ def hello_world_child(job, hw):
     # can also use:  job.updateGlobalFile() given the FileStoreID instantiation.
     job.fileStore.writeGlobalFile('bar_bam.txt')
 
+    os.remove('bar_bam.txt')
+
 
 def main():
     # Boilerplate -- startToil requires options
 
     parser = argparse.ArgumentParser()
     Job.Runner.addToilOptions(parser)
-    args = parser.parse_args()
+    options = parser.parse_args()
 
     # Create object that contains our FileStoreIDs
 
-    print args.jobStore
     # Launch first toil Job
     i = Job.wrapJobFn(hello_world, memory=100, cores=0.5, disk="3G")
-    Job.Runner.startToil(i, args)
+    with Toil(options) as toil:
+        toil.start(i)
 
 
 if __name__ == '__main__':
diff --git a/src/toil/test/src/regularLogTest.py b/src/toil/test/src/regularLogTest.py
new file mode 100644
--- /dev/null
+++ b/src/toil/test/src/regularLogTest.py
@@ -0,0 +1,45 @@
+# Copyright (C) 2015 UCSC Computational Genomics Lab
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+import subprocess
+import sys
+from toil.test import ToilTest
+from toil.test.mesos import helloWorld
+
+
+class RegularLogTest(ToilTest):
+
+    def testLogToMaster(self):
+        toilOutput = subprocess.check_output([sys.executable,
+                                              '-m', helloWorld.__name__,
+                                              './toilTest',
+                                              '--clean=always',
+                                              '--logLevel=info'], stderr=subprocess.STDOUT)
+        assert helloWorld.childMessage in toilOutput
+
+    def testMultipleLogToMaster(self):
+        toilOutput = subprocess.check_output([sys.executable,
+                                              '-m', helloWorld.__name__,
+                                              './toilTest',
+                                              '--clean=always',
+                                              '--logLevel=info'], stderr=subprocess.STDOUT)
+        assert helloWorld.parentMessage in toilOutput
+
+    def testRegularLog(self):
+        toilOutput = subprocess.check_output([sys.executable,
+                                              '-m', helloWorld.__name__,
+                                              './toilTest',
+                                              '--clean=always',
+                                              '--batchSystem=singleMachine',
+                                              '--logLevel=info'], stderr=subprocess.STDOUT)
+        assert "single machine batch system" in toilOutput