Repository: DataBiosphere/toil

Issue:
Add "failed_" prefix to failed logs
With #2781 merged, we can now enable writing all logs from all jobs. However, currently logs from failed jobs have the same format/structure as successful ones, so it makes it difficult to find which ones have failed (one can look at the `###.log` suffix for jobs that have been retried, but it's still inconvenient and won't work with `--retryCount 0`).

Proposal is to add a `failed_` prefix to failed logs.

┆Issue is synchronized with this [Jira Task](https://ucsc-cgl.atlassian.net/browse/TOIL-417)
┆Issue Number: TOIL-417



Discussion:

Resolving patch:
diff --git a/src/toil/leader.py b/src/toil/leader.py
--- a/src/toil/leader.py
+++ b/src/toil/leader.py
@@ -793,7 +793,7 @@ def processFinishedJob(self, batchSystemID, resultStatus, wallTime=None):
                                                       message='The job seems to have left a log file, indicating failure: %s' % jobGraph)
                 if self.config.writeLogs or self.config.writeLogsGzip:
                     with jobGraph.getLogFileHandle(self.jobStore) as logFileStream:
-                        StatsAndLogging.writeLogFiles(jobGraph.chainedJobs, logFileStream, self.config)
+                        StatsAndLogging.writeLogFiles(jobGraph.chainedJobs, logFileStream, self.config, failed=True)
             if resultStatus != 0:
                 # If the batch system returned a non-zero exit code then the worker
                 # is assumed not to have captured the failure of the job, so we
@@ -823,7 +823,7 @@ def processFinishedJob(self, batchSystemID, resultStatus, wallTime=None):
                                         jobNames = [str(jobGraph)]
                                     jobNames = [jobName + '_' + batchSystemFileRoot for jobName in jobNames]
                                     batchSystemFileStream.seek(0)
-                                    StatsAndLogging.writeLogFiles(jobNames, batchSystemFileStream, self.config)
+                                    StatsAndLogging.writeLogFiles(jobNames, batchSystemFileStream, self.config, failed=True)
                             else:
                                 logger.warn('The batch system left an empty file %s' % batchSystemFile)
 
diff --git a/src/toil/statsAndLogging.py b/src/toil/statsAndLogging.py
--- a/src/toil/statsAndLogging.py
+++ b/src/toil/statsAndLogging.py
@@ -57,13 +57,15 @@ def logWithFormatting(cls, jobStoreID, jobLogs, method=logger.debug, message=Non
             method('%s    %s', jobStoreID, line.rstrip('\n'))
 
     @classmethod
-    def writeLogFiles(cls, jobNames, jobLogList, config):
-        def createName(logPath, jobName, logExtension):
+    def writeLogFiles(cls, jobNames, jobLogList, config, failed=False):
+        def createName(logPath, jobName, logExtension, failed=False):
             logName = jobName.replace('-', '--')
             logName = logName.replace('/', '-')
             logName = logName.replace(' ', '_')
             logName = logName.replace("'", '')
             logName = logName.replace('"', '')
+            # Add a "failed_" prefix to logs from failed jobs.
+            logName = ('failed_' if failed else '') + logName
             counter = 0
             while True:
                 suffix = str(counter).zfill(3) + logExtension
@@ -89,7 +91,7 @@ def createName(logPath, jobName, logExtension):
             # we don't have anywhere to write the logs, return now
             return
 
-        fullName = createName(path, mainFileName, extension)
+        fullName = createName(path, mainFileName, extension, failed)
         with writeFn(fullName, 'wb') as f:
             for l in jobLogList:
                 try:
@@ -102,7 +104,7 @@ def createName(logPath, jobName, logExtension):
         for alternateName in jobNames[1:]:
             # There are chained jobs in this output - indicate this with a symlink
             # of the job's name to this file
-            name = createName(path, alternateName, extension)
+            name = createName(path, alternateName, extension, failed)
             os.symlink(os.path.relpath(fullName, path), name)
 
     @classmethod


Test patch: