Repository: DataBiosphere/toil

Issue:
Add an option to log output from all jobs without --logDebug
It's often useful to see the logs from all jobs for debugging. Currently, the only way to turn this on is through `--logDebug`, which enables all kinds of other debug logging and is too noisy.

We can add a new option, `--writeLogsFromAllJobs`, to enable logging from all jobs without enabling debug mode.

┆Issue is synchronized with this [Jira Task](https://ucsc-cgl.atlassian.net/browse/TOIL-413)
┆Issue Number: TOIL-413



Discussion:

Resolving patch:
diff --git a/src/toil/common.py b/src/toil/common.py
--- a/src/toil/common.py
+++ b/src/toil/common.py
@@ -123,6 +123,7 @@ def __init__(self):
         self.maxLogFileSize = 64000
         self.writeLogs = None
         self.writeLogsGzip = None
+        self.writeLogsFromAllJobs = False
         self.sseKey = None
         self.cseKey = None
         self.servicePollingInterval = 60
@@ -270,8 +271,14 @@ def parseIntList(s):
         setOption("maxLogFileSize", h2b, iC(1))
         setOption("writeLogs")
         setOption("writeLogsGzip")
+        setOption("writeLogsFromAllJobs")
         setOption("runCwlInternalJobsOnWorkers")
 
+        assert not (self.writeLogs and self.writeLogsGzip), \
+            "Cannot use both --writeLogs and --writeLogsGzip at the same time."
+        assert not self.writeLogsFromAllJobs or self.writeLogs or self.writeLogsGzip, \
+            "To enable --writeLogsFromAllJobs, either --writeLogs or --writeLogsGzip must be set."
+
         def checkSse(sseKey):
             with open(sseKey) as f:
                 assert (len(f.readline().rstrip()) == 32)
@@ -545,13 +552,18 @@ def _addOptions(addGroupFn, config):
                      "specified path. Any non-empty standard output and error from failed batch "
                      "system jobs will also be written into files at this path. "
                      "The current working directory will be used if a path is "
-                     "not specified explicitly. Note: By default "
-                     "only the logs of failed jobs are returned to leader. Set log level to "
-                     "'debug' to get logs back from successful jobs, and adjust 'maxLogFileSize' "
-                     "to control the truncation limit for worker logs.")
+                     "not specified explicitly. Note: By default only the logs of failed jobs are "
+                     "returned to leader. Set log level to 'debug' or enable "
+                     "'--writeLogsFromAllJobs' to get logs back from successful jobs, and adjust "
+                     "'maxLogFileSize' to control the truncation limit for worker logs.")
     addOptionFn("--writeLogsGzip", dest="writeLogsGzip", nargs='?', action='store',
                 default=None, const=os.getcwd(),
                 help="Identical to --writeLogs except the logs files are gzipped on the leader.")
+    addOptionFn("--writeLogsFromAllJobs", dest="writeLogsFromAllJobs", action='store_true',
+                default=False,
+                help="Whether to write logs from all jobs (including the successful ones) without "
+                     "necessarily setting the log level to 'debug'. Ensure that either --writeLogs "
+                     "or --writeLogsGzip is set if enabling this option.")
     addOptionFn("--realTimeLogging", dest="realTimeLogging", action="store_true", default=False,
                 help="Enable real-time logging from workers to masters")
 
diff --git a/src/toil/statsAndLogging.py b/src/toil/statsAndLogging.py
--- a/src/toil/statsAndLogging.py
+++ b/src/toil/statsAndLogging.py
@@ -78,9 +78,6 @@ def createName(logPath, jobName, logExtension):
         mainFileName = jobNames[0]
         extension = '.log'
 
-        assert not (config.writeLogs and config.writeLogsGzip), \
-            "Cannot use both --writeLogs and --writeLogsGzip at the same time."
-
         if config.writeLogs:
             path = config.writeLogs
             writeFn = open
diff --git a/src/toil/worker.py b/src/toil/worker.py
--- a/src/toil/worker.py
+++ b/src/toil/worker.py
@@ -40,6 +40,11 @@
 from toil.lib.bioio import getTotalCpuTime
 from toil.lib.bioio import getTotalCpuTimeAndMemoryUsage
 from toil.deferred import DeferredFunctionManager
+try:
+    from toil.cwl.cwltoil import CWL_INTERNAL_JOBS
+except ImportError:
+    # CWL extra not installed
+    CWL_INTERNAL_JOBS = ()
 
 
 logging.basicConfig()
@@ -489,7 +494,8 @@ def workerScript(jobStore, config, jobName, jobStoreID, redirectOutputToLogFile=
                 w.write(f.read().encode('utf-8')) # TODO load file using a buffer
         jobStore.update(jobGraph)
 
-    elif debugging and redirectOutputToLogFile:  # write log messages
+    elif ((debugging or (config.writeLogsFromAllJobs and not jobName.startswith(CWL_INTERNAL_JOBS)))
+          and redirectOutputToLogFile):  # write log messages
         with open(tempWorkerLogPath, 'r') as logFile:
             if os.path.getsize(tempWorkerLogPath) > logFileByteReportLimit != 0:
                 if logFileByteReportLimit > 0:


Test patch: