Repository: DataBiosphere/toil

Issue:
Report jobs' accessed files when they fail
We should log each job's accesses to the file store. If the job fails, we should report the files that it downloaded and how to get at them, and (if available) what names they were downloaded as.

This would help a lot with debugging workflows, because you could then pull the files yourself and try any shell commands that failed on them.

This is part of #1855.

┆Issue is synchronized with this [Jira Story](https://ucsc-cgl.atlassian.net/browse/TOIL-2)
┆Issue Number: TOIL-2



Discussion:

Resolving patch:
diff --git a/src/toil/fileStores/abstractFileStore.py b/src/toil/fileStores/abstractFileStore.py
--- a/src/toil/fileStores/abstractFileStore.py
+++ b/src/toil/fileStores/abstractFileStore.py
@@ -19,6 +19,7 @@
 import logging
 import os
 import tempfile
+from typing import Union
 
 from toil.lib.objects import abstractclassmethod
 from toil.lib.misc import WriteWatchingStream
@@ -93,6 +94,9 @@ def __init__(self, jobStore, jobGraph, localTempDir, waitForPreviousCommit):
         # running job is cleaned up.
         # May be modified by the worker to actually delete jobs!
         self.jobsToDelete = set()
+        # Holds records of file ID, or file ID and local path, for reporting
+        # the accessed files of failed jobs.
+        self._accessLog = []
 
     @staticmethod
     def createFileStore(jobStore, jobGraph, localTempDir, waitForPreviousCommit, caching):
@@ -131,17 +135,27 @@ def shutdownFileStore(workflowDir, workflowID):
             # This absence of cacheDir suggests otherwise.
             NonCachingFileStore.shutdown(workflowDir)
 
-    @abstractmethod
     @contextmanager
     def open(self, job):
         """
         The context manager used to conduct tasks prior-to, and after a job has
         been run. File operations are only permitted inside the context
         manager.
+        
+        Implementations must only yield from within `with super().open(job):`.
 
         :param toil.job.Job job: The job instance of the toil job to run.
         """
-        raise NotImplementedError()
+        
+        failed = True
+        try:
+            yield
+            failed = False
+        finally:
+            # Do a finally instead of an except/raise because we don't want
+            # to appear as "another exception occurred" in the stack trace.
+            if failed:
+                self._dumpAccessLogs()
 
     # Functions related to temp files and directories
     def getLocalTempDir(self):
@@ -243,6 +257,39 @@ def handle(numBytes):
             
             yield wrappedStream, fileID
 
+    def _dumpAccessLogs(self):
+        """
+        When something goes wrong, log a report of the files that were accessed
+        while the file store was open.
+        """
+        
+        if len(self._accessLog) > 0:
+            logger.warning('Failed job accessed files:')
+        
+            for item in self._accessLog:
+                # For each access record
+                if len(item) == 2:
+                    # If it has a name, dump wit the name
+                    logger.warning('Downloaded file \'%s\' to path \'%s\'', *item)
+                else:
+                    # Otherwise dump without the name
+                    logger.warning('Streamed file \'%s\'', *item)
+                    
+    def logAccess(self, fileStoreID: Union[FileID, str], destination: Union[str, None] = None):
+        """
+        Record that the given file was read by the job, to be announced if the
+        job fails. If destination is not None, it gives the path that the file
+        was downloaded to. Otherwise, assumes that the file was streamed.
+        
+        Must be called by :meth:`readGlobalFile` and :meth:`readGlobalFileStream`
+        implementations.
+        """
+        
+        if destination is not None:
+            self._accessLog.append((fileStoreID, destination))
+        else:
+            self._accessLog.append((fileStoreID,))
+
     @abstractmethod
     def readGlobalFile(self, fileStoreID, userPath=None, cache=True, mutable=False, symlink=False):
         """
@@ -256,6 +303,8 @@ def readGlobalFile(self, fileStoreID, userPath=None, cache=True, mutable=False,
 
         The destination file must not be deleted by the user; it can only be
         deleted through deleteLocalFile.
+        
+        Implementations must call :meth:`logAccess` to report the download.
 
         :param toil.fileStores.FileID or str fileStoreID: job store id for the file
         :param string userPath: a path to the name of file to which the global file will be copied
@@ -273,6 +322,8 @@ def readGlobalFileStream(self, fileStoreID):
         Similar to readGlobalFile, but allows a stream to be read from the job store. The yielded
         file handle does not need to and should not be closed explicitly.
 
+        Implementations must call :meth:`logAccess` to report the download.
+
         :return: a context manager yielding a file handle which can be read from.
         """
         raise NotImplementedError()
diff --git a/src/toil/fileStores/cachingFileStore.py b/src/toil/fileStores/cachingFileStore.py
--- a/src/toil/fileStores/cachingFileStore.py
+++ b/src/toil/fileStores/cachingFileStore.py
@@ -992,7 +992,8 @@ def open(self, job):
         self._allocateSpaceForJob(self.jobDiskBytes)
         try:
             os.chdir(self.localTempDir)
-            yield
+            with super().open(job):
+                yield
         finally:
             # See how much disk space is used at the end of the job.
             # Not a real peak disk usage, but close enough to be useful for warning the user.
@@ -1116,12 +1117,16 @@ def readGlobalFile(self, fileStoreID, userPath=None, cache=True, mutable=False,
             # We want to use the cache
 
             if mutable:
-                return self._readGlobalFileMutablyWithCache(fileStoreID, localFilePath, readerID)
+                finalPath = self._readGlobalFileMutablyWithCache(fileStoreID, localFilePath, readerID)
             else:
-                return self._readGlobalFileWithCache(fileStoreID, localFilePath, symlink, readerID)
+                finalPath = self._readGlobalFileWithCache(fileStoreID, localFilePath, symlink, readerID)
         else:
             # We do not want to use the cache
-            return self._readGlobalFileWithoutCache(fileStoreID, localFilePath, mutable, symlink, readerID)
+            finalPath = self._readGlobalFileWithoutCache(fileStoreID, localFilePath, mutable, symlink, readerID)
+            
+        # Record access in case the job crashes and we have to log it
+        self.logAccess(fileStoreID, finalPath)
+        return finalPath
 
 
     def _readGlobalFileWithoutCache(self, fileStoreID, localFilePath, mutable, symlink, readerID):
@@ -1598,7 +1603,9 @@ def readGlobalFileStream(self, fileStoreID):
         if str(fileStoreID) in self.filesToDelete:
             # File has already been deleted
             raise FileNotFoundError('Attempted to read deleted file: {}'.format(fileStoreID))
-
+        
+        self.logAccess(fileStoreID)
+        
         # TODO: can we fulfil this from the cache if the file is in the cache?
         # I think we can because if a job is keeping the file data on disk due to having it open, it must be paying for it itself.
         return self.jobStore.readFileStream(fileStoreID)
diff --git a/src/toil/fileStores/nonCachingFileStore.py b/src/toil/fileStores/nonCachingFileStore.py
--- a/src/toil/fileStores/nonCachingFileStore.py
+++ b/src/toil/fileStores/nonCachingFileStore.py
@@ -64,7 +64,8 @@ def open(self, job):
                            self.jobName)
         try:
             os.chdir(self.localTempDir)
-            yield
+            with super().open(job):
+                yield
         finally:
             diskUsed = getDirSizeRecursively(self.localTempDir)
             logString = ("Job {jobName} used {percent:.2f}% ({humanDisk}B [{disk}B] used, "
@@ -105,11 +106,13 @@ def readGlobalFile(self, fileStoreID, userPath=None, cache=True, mutable=False,
 
         self.jobStore.readFile(fileStoreID, localFilePath, symlink=symlink)
         self.localFileMap[fileStoreID].append(localFilePath)
+        self.logAccess(fileStoreID, localFilePath)
         return localFilePath
 
     @contextmanager
     def readGlobalFileStream(self, fileStoreID):
         with self.jobStore.readFileStream(fileStoreID) as f:
+            self.logAccess(fileStoreID)
             yield f
 
     def exportFile(self, jobStoreFileID, dstUrl):


Test patch:
diff --git a/src/toil/test/src/fileStoreTest.py b/src/toil/test/src/fileStoreTest.py
--- a/src/toil/test/src/fileStoreTest.py
+++ b/src/toil/test/src/fileStoreTest.py
@@ -117,6 +117,53 @@ def testToilIsNotBroken(self):
             C.addChild(D)
             Job.Runner.startToil(A, self.options)
 
+        @slow
+        def testFileStoreLogging(self):
+            """
+            Write a couple of files to the jobstore.  Delete a couple of them.  Read back written
+            and locally deleted files.
+            """
+            
+            class WatchingHandler(logging.Handler):
+                """
+                A logging handler that watches for a certain substring and
+                trips a flag if it appears.
+                """
+                def __init__(self, match: str):
+                    super().__init__()
+                    self.match = match
+                    self.seen = False
+                def emit(self, record):
+                    if self.match in record.getMessage():
+                        self.seen = True
+            
+            handler = WatchingHandler("cats.txt")
+            
+            logging.getLogger().addHandler(handler)
+            
+            F = Job.wrapJobFn(self._accessAndFail,
+                              disk='100M')
+            try:
+                Job.Runner.startToil(F, self.options)
+            except FailedJobsException:
+                # We expect this.
+                pass
+            
+            logging.getLogger().removeHandler(handler)
+            
+            assert handler.seen, "Downloaded file name not found in logs of failing Toil run"
+            
+        @staticmethod
+        def _accessAndFail(job):
+            with job.fileStore.writeGlobalFileStream() as (stream, fileID):
+                stream.write('Cats'.encode('utf-8'))
+            localPath = os.path.join(job.fileStore.getLocalTempDir(), 'cats.txt')
+            job.fileStore.readGlobalFile(fileID, localPath)
+            with job.fileStore.readGlobalFileStream(fileID) as stream2:
+                pass
+            raise RuntimeError("I do not like this file")
+            
+
         # Test filestore operations.  This is a slightly less intense version of the cache specific
         # test `testReturnFileSizes`
         @slow