Repository: DataBiosphere/toil

Issue:
If log level is DEBUG, send all worker output back to master
... regardless of whether a failure occurred.



Discussion:
Do you mind if CJ takes this on? He needs more work.

Please, be my guest! Sorry to not have got it done earlier.

On Thu, Aug 27, 2015 at 12:41 PM, Hannes Schmidt notifications@github.com
wrote:

> Do you mind if CJ takes this on? He needs more work.
> 
> —
> Reply to this email directly or view it on GitHub
> https://github.com/BD2KGenomics/toil/issues/301#issuecomment-135532625.
> 
> ## 
> 
> You received this message because you are subscribed to the Google Groups
> "cgloud-jenkins" group.
> To unsubscribe from this group and stop receiving emails from it, send an
> email to cgloud-jenkins+unsubscribe@googlegroups.com.
> To post to this group, send email to cgloud-jenkins@googlegroups.com.
> To view this discussion on the web visit
> https://groups.google.com/d/msgid/cgloud-jenkins/BD2KGenomics/toil/issues/301/135532625%40github.com
> https://groups.google.com/d/msgid/cgloud-jenkins/BD2KGenomics/toil/issues/301/135532625%40github.com?utm_medium=email&utm_source=footer
> .
> 
> For more options, visit https://groups.google.com/d/optout.


Resolving patch:
diff --git a/src/toil/leader.py b/src/toil/leader.py
--- a/src/toil/leader.py
+++ b/src/toil/leader.py
@@ -50,9 +50,13 @@ def statsAndLoggingAggregatorProcess(jobStore, stop):
         #Call back function
         def statsAndLoggingCallBackFn(fileHandle2):
             node = ET.parse(fileHandle2).getroot()
-            for message in node.find("messages").findall("message"):
+            nodesNamed = node.find("messages").findall
+            for message in nodesNamed("message"):
                 logger.warn("Got message from job at time: %s : %s",
                                     time.strftime("%m-%d-%Y %H:%M:%S"), message.text)
+            for log in nodesNamed("log"):
+                logger.info("%s:     %s" %
+                                    tuple(log.text.split("!",1)))# the jobID is separated from log by "!"
             ET.ElementTree(node).write(fileHandle)
         
         #The main loop
diff --git a/src/toil/worker.py b/src/toil/worker.py
--- a/src/toil/worker.py
+++ b/src/toil/worker.py
@@ -165,11 +165,15 @@ def main():
     #the file descriptor out from under it.
     logger.addHandler(logging.StreamHandler(sys.stderr))
 
+    debugging = logging.getLogger().isEnabledFor(logging.DEBUG)
     ##########################################
     #Worker log file trapped from here on in
     ##########################################
 
     workerFailed = False
+    elementNode = ET.Element("worker")
+    messageNode = ET.SubElement(elementNode, "messages")
+    messages = []
     try:
 
         #Put a message at the top of the log, just to make sure it's working.
@@ -216,9 +220,6 @@ def main():
         if config.stats:
             startTime = time.time()
             startClock = getTotalCpuTime()
-            stats = ET.Element("worker")
-        else:
-            stats = None
 
         startTime = time.time() 
         while True:
@@ -233,7 +234,7 @@ def main():
                     
                     #Is a job command
                     messages = Job._loadJob(job.command, jobStore)._execute( jobWrapper=job,
-                                    stats=stats, localTempDir=localTempDir, 
+                                    stats=elementNode if config.stats else None, localTempDir=localTempDir,
                                     jobStore=jobStore)
                     
                     #Remove the temporary file directory
@@ -241,12 +242,10 @@ def main():
     
                 else: #Is another command (running outside of jobs may be deprecated)
                     system(job.command)
-                    messages = []
             else:
                 #The command may be none, in which case
                 #the job is just a shell ready to be deleted
                 assert len(job.stack) == 0
-                messages = []
                 break
             
             ##########################################
@@ -321,22 +320,13 @@ def main():
         ##########################################
         #Finish up the stats
         ##########################################
-
-        if stats != None:
+        if config.stats:
             totalCPUTime, totalMemoryUsage = getTotalCpuTimeAndMemoryUsage()
-            stats.attrib["time"] = str(time.time() - startTime)
-            stats.attrib["clock"] = str(totalCPUTime - startClock)
-            stats.attrib["memory"] = str(totalMemoryUsage)
-            m = ET.SubElement(stats, "messages")
-            for message in messages:
-                ET.SubElement(m, "message").text = message
-            jobStore.writeStatsAndLogging(ET.tostring(stats))
-        elif len(messages) > 0: #No stats, but still need to report log messages
-            l = ET.Element("worker")
-            m = ET.SubElement(l, "messages")
-            for message in messages:
-                ET.SubElement(m, "message").text = message
-            jobStore.writeStatsAndLogging(ET.tostring(l))
+            elementNode.attrib["time"] = str(time.time() - startTime)
+            elementNode.attrib["clock"] = str(totalCPUTime - startClock)
+            elementNode.attrib["memory"] = str(totalMemoryUsage)
+        for message in messages:
+            ET.SubElement(messageNode, "message").text = message
         
         logger.info("Finished running the chain of jobs on this node, we ran for a total of %f seconds", time.time() - startTime)
     
@@ -384,6 +374,15 @@ def main():
         job.setLogFile(tempWorkerLogPath, jobStore)
         os.remove(tempWorkerLogPath)
         jobStore.update(job)
+    elif debugging: # write log messages
+        truncateFile(tempWorkerLogPath)
+        with open(tempWorkerLogPath, 'r') as logFile:
+            logMessages = logFile.read().splitlines()
+        for logMessage in logMessages:
+            ET.SubElement(messageNode, "log").text = jobStoreID+"!"+logMessage
+
+    if (debugging or config.stats or messages) and not workerFailed: # We have stats/logging to report back
+        jobStore.writeStatsAndLogging(ET.tostring(elementNode))
 
     #Remove the temp dir
     shutil.rmtree(localWorkerTempDir)


Test patch: