Repository: DataBiosphere/toil

Issue:
Caching hangs on harbinger file in sortTest
http://jenkins.cgcloud.info/job/toil-pull-requests/1413/

strace of the worker process

```
Trace of process 10637 - /mnt/ephemeral/workspace/toil-pull-requests/venv/bin/python /mnt/ephemeral/workspace/toil-pull-requests/venv/bin/_toil_worker /mnt/ephemeral/tmp/toil-test-toil.test.utils.utilsTe
Process 10637 attached
select(0, NULL, NULL, NULL, {2, 824883}) = 0 (Timeout)
flock(11, LOCK_EX)                      = 0
stat("/mnt/ephemeral/tmp/toil-43132b5e-c11d-4a83-aae6-8d302f973781/cache-43132b5e-c11d-4a83-aae6-8d302f973781/.TS9kL3RtcF80N1dGUC50bXA=.harbinger", {st_mode=S_IFREG|0664, st_size=0, ...}) = 0
flock(11, LOCK_UN)                      = 0
select(0, NULL, NULL, NULL, {20, 0})    = 0 (Timeout)
flock(11, LOCK_EX)                      = 0
stat("/mnt/ephemeral/tmp/toil-43132b5e-c11d-4a83-aae6-8d302f973781/cache-43132b5e-c11d-4a83-aae6-8d302f973781/.TS9kL3RtcF80N1dGUC50bXA=.harbinger", {st_mode=S_IFREG|0664, st_size=0, ...}) = 0
flock(11, LOCK_UN)                      = 0
select(0, NULL, NULL, NULL, {20, 0})    = 0 (Timeout)
flock(11, LOCK_EX)                      = 0
stat("/mnt/ephemeral/tmp/toil-43132b5e-c11d-4a83-aae6-8d302f973781/cache-43132b5e-c11d-4a83-aae6-8d302f973781/.TS9kL3RtcF80N1dGUC50bXA=.harbinger", {st_mode=S_IFREG|0664, st_size=0, ...}) = 0
flock(11, LOCK_UN)                      = 0
select(0, NULL, NULL, NULL, {20, 0})    = 0 (Timeout)
flock(11, LOCK_EX)                      = 0
stat("/mnt/ephemeral/tmp/toil-43132b5e-c11d-4a83-aae6-8d302f973781/cache-43132b5e-c11d-4a83-aae6-8d302f973781/.TS9kL3RtcF80N1dGUC50bXA=.harbinger", {st_mode=S_IFREG|0664, st_size=0, ...}) = 0
flock(11, LOCK_UN)                      = 0
select(0, NULL, NULL, NULL, {20, 0})    = 0 (Timeout)
flock(11, LOCK_EX)                      = 0
stat("/mnt/ephemeral/tmp/toil-43132b5e-c11d-4a83-aae6-8d302f973781/cache-43132b5e-c11d-4a83-aae6-8d302f973781/.TS9kL3RtcF80N1dGUC50bXA=.harbinger", {st_mode=S_IFREG|0664, st_size=0, ...}) = 0
flock(11, LOCK_UN)                      = 0
select(0, NULL, NULL, NULL, {20, 0})    = 0 (Timeout)
flock(11, LOCK_EX)                      = 0
stat("/mnt/ephemeral/tmp/toil-43132b5e-c11d-4a83-aae6-8d302f973781/cache-43132b5e-c11d-4a83-aae6-8d302f973781/.TS9kL3RtcF80N1dGUC50bXA=.harbinger", {st_mode=S_IFREG|0664, st_size=0, ...}) = 0
flock(11, LOCK_UN)                      = 0
select(0, NULL, NULL, NULL, {20, 0})    = 0 (Timeout)
flock(11, LOCK_EX)                      = 0
stat("/mnt/ephemeral/tmp/toil-43132b5e-c11d-4a83-aae6-8d302f973781/cache-43132b5e-c11d-4a83-aae6-8d302f973781/.TS9kL3RtcF80N1dGUC50bXA=.harbinger", {st_mode=S_IFREG|0664, st_size=0, ...}) = 0
flock(11, LOCK_UN)                      = 0
select(0, NULL, NULL, NULL, {20, 0})    = 0 (Timeout)
flock(11, LOCK_EX)                      = 0
stat("/mnt/ephemeral/tmp/toil-43132b5e-c11d-4a83-aae6-8d302f973781/cache-43132b5e-c11d-4a83-aae6-8d302f973781/.TS9kL3RtcF80N1dGUC50bXA=.harbinger", {st_mode=S_IFREG|0664, st_size=0, ...}) = 0
flock(11, LOCK_UN)                      = 0
select(0, NULL, NULL, NULL, {20, 0})    = 0 (Timeout)
flock(11, LOCK_EX)                      = 0
stat("/mnt/ephemeral/tmp/toil-43132b5e-c11d-4a83-aae6-8d302f973781/cache-43132b5e-c11d-4a83-aae6-8d302f973781/.TS9kL3RtcF80N1dGUC50bXA=.harbinger", {st_mode=S_IFREG|0664, st_size=0, ...}) = 0
flock(11, LOCK_UN)                      = 0
select(0, NULL, NULL, NULL, {20, 0})    = 0 (Timeout)
flock(11, LOCK_EX)                      = 0
stat("/mnt/ephemeral/tmp/toil-43132b5e-c11d-4a83-aae6-8d302f973781/cache-43132b5e-c11d-4a83-aae6-8d302f973781/.TS9kL3RtcF80N1dGUC50bXA=.harbinger", {st_mode=S_IFREG|0664, st_size=0, ...}) = 0
flock(11, LOCK_UN)                      = 0
select(0, NULL, NULL, NULL, {20, 0})    = 0 (Timeout)
flock(11, LOCK_EX)                      = 0
stat("/mnt/ephemeral/tmp/toil-43132b5e-c11d-4a83-aae6-8d302f973781/cache-43132b5e-c11d-4a83-aae6-8d302f973781/.TS9kL3RtcF80N1dGUC50bXA=.harbinger", {st_mode=S_IFREG|0664, st_size=0, ...}) = 0
flock(11, LOCK_UN)                      = 0
select(0, NULL, NULL, NULL, {20, 0})    = 0 (Timeout)
flock(11, LOCK_EX)                      = 0
stat("/mnt/ephemeral/tmp/toil-43132b5e-c11d-4a83-aae6-8d302f973781/cache-43132b5e-c11d-4a83-aae6-8d302f973781/.TS9kL3RtcF80N1dGUC50bXA=.harbinger", {st_mode=S_IFREG|0664, st_size=0, ...}) = 0
flock(11, LOCK_UN)                      = 0
```

That's all I have.

AbstractCacheTest._multipleFileReader docstring is incorrect



Discussion:

Resolving patch:
diff --git a/src/toil/job.py b/src/toil/job.py
--- a/src/toil/job.py
+++ b/src/toil/job.py
@@ -1173,17 +1173,27 @@ def readGlobalFile(self, fileStoreID, userPath=None, cache=True, mutable=None):
                         self.returnFileSize(fileStoreID, localFilePath, lockFileHandle,
                                             fileAlreadyCached=True)
                 # If the file is not in cache, check whether the .harbinger file for the given
-                # FileStoreID exists. If it does, the wait and periodically check for the removal of
-                # the file and the addition of the completed download into cache of the file by the
-                # other job. Then we link to it.
+                # FileStoreID exists.  If it does, the wait and periodically check for the removal
+                # of the file and the addition of the completed download into cache of the file by
+                # the other job. Then we link to it.
                 elif fileIsLocal and os.path.exists(harbingerFileName):
                     logger.info('CACHE: Waiting for another worker to download file with ID %s.'
                                 % fileStoreID)
                     while os.path.exists(harbingerFileName):
-                        # Release the file lock and then periodically check for completed download
-                        flock(lockFileHandle, LOCK_UN)
-                        time.sleep(20)  # What should this value be?
-                        flock(lockFileHandle, LOCK_EX)
+                        # Ensure that the process downloading the file is still alive.  The PID will
+                        # be in the harbinger file.
+                        pid = int(open(harbingerFileName).read())
+                        if self._pidExists(pid):
+                            # Release the file lock and then wait for a bit before repeating.
+                            flock(lockFileHandle, LOCK_UN)
+                            time.sleep(20)
+                            # Grab the file lock before repeating.
+                            flock(lockFileHandle, LOCK_EX)
+                        else:
+                            # The process that was supposed to download the file has died so we need
+                            # to remove the harbinger.
+                            os.remove(harbingerFileName)
+
                     # If the code reaches here, the partial lock file has been removed. This means
                     # either the file was successfully downloaded and added to cache, or something
                     # failed. To prevent code duplication, we recursively call readGlobalFile.
@@ -1195,8 +1205,12 @@ def readGlobalFile(self, fileStoreID, userPath=None, cache=True, mutable=None):
                     logger.debug('CACHE: Cache miss on file with ID \'%s\'.' % fileStoreID)
                     if fileIsLocal and cache:
                         # If caching of the downloaded file is desired, First create the .partial
-                        # file so other jobs know not to redundantly download the same file.
-                        open(harbingerFileName, 'w').close()  # This emulates the system 'touch'
+                        # file so other jobs know not to redundantly download the same file.  Write
+                        # the PID of this process into the file so other jobs know who is carrying
+                        # out the download.
+                        with open(harbingerFileName + '.tmp', 'w') as harbingerFile:
+                            harbingerFile.write(str(os.getpid()))
+                        os.rename(harbingerFileName + '.tmp', harbingerFileName)
                         # Now release the file lock while the file is downloaded as download could
                         # take a while.
                         flock(lockFileHandle, LOCK_UN)
@@ -1205,13 +1219,13 @@ def readGlobalFile(self, fileStoreID, userPath=None, cache=True, mutable=None):
                         try:
                             self.jobStore.readFile(fileStoreID,
                                                    '/.'.join(os.path.split(cachedFileName)))
-                        except: #TODO
+                        except:
                             if os.path.exists('/.'.join(os.path.split(cachedFileName))):
                                 os.remove('/.'.join(os.path.split(cachedFileName)))
+                            raise
                         else:
                             # If the download succeded, officially add the file to cache (by
-                            # recording it in the cache lock file) if possible. Possibly use better
-                            # function name.
+                            # recording it in the cache lock file) if possible.
                             if os.path.exists('/.'.join(os.path.split(cachedFileName))):
                                 os.rename('/.'.join(os.path.split(cachedFileName)), cachedFileName)
                                 self.addToCache(localFilePath, fileStoreID, 'read', mutable)
@@ -1913,6 +1927,29 @@ def updateJobReqs(self, fileSize, actions):
                 # and it is added if the file was removed form the cache.
                 self.jobReqs -= (fileSize * multiplier)
 
+        @staticmethod
+        def _pidExists(pid):
+            """
+            This will return True if the process associated with pid is still running on the
+            machine.
+            This is based on stackoverflow question 568271.
+
+            :param int pid: ID of the process to check for
+            :return: True/False
+            :rtype: bool
+            """
+            assert pid > 0
+            try:
+                os.kill(pid, 0)
+            except OSError as err:
+                if err.errno == errno.ESRCH:
+                    # ESRCH == No such process
+                    return False
+                else:
+                    raise
+            else:
+                return True
+
     class Service:
         """
         Abstract class used to define the interface to a service.


Test patch:
diff --git a/src/toil/test/src/jobCacheTest.py b/src/toil/test/src/jobCacheTest.py
--- a/src/toil/test/src/jobCacheTest.py
+++ b/src/toil/test/src/jobCacheTest.py
@@ -488,7 +488,7 @@ def _testMultipleJobsReadGlobalFileFunction(self, cacheHit):
             B = Job.wrapJobFn(self._probeJobReqs, sigmaJob=100, disk='100M')
             jobs = {}
             for i in xrange(0, 10):
-                jobs[i] = Job.wrapJobFn(self._multipleFileReader, diskMB=1024, fileInfo=A.rv(),
+                jobs[i] = Job.wrapJobFn(self._multipleFileReader, diskMB=1024, fsID=A.rv(),
                                         maxWriteFile=os.path.abspath(x.name), disk='1G',
                                         memory='10M', cores=1)
                 A.addChild(jobs[i])
@@ -498,18 +498,19 @@ def _testMultipleJobsReadGlobalFileFunction(self, cacheHit):
                 assert int(y.read()) > 2
 
         @staticmethod
-        def _multipleFileReader(job, diskMB, fileInfo, maxWriteFile):
+        def _multipleFileReader(job, diskMB, fsID, maxWriteFile):
             """
-            Read fsID from file store and add to cache.  Assert cached file size in the cache
-            lock file never goes up, assert sum of job reqs is always (a multiple of job reqs) -
-            (number of files linked to the cachedfile * filesize).
+            Read a file from the job store immutable and explicitly ask to have it in the cache.
+            If we are using the File Job Store, assert sum of cached file sizes in the cache lock
+            file is zero, else assert it is equal to the read file.
+            Also assert the sum job reqs + (number of files linked to the cachedfile * filesize) is
+            and integer multiple of the disk requirements provided to this job.
 
             :param int diskMB: disk requirements provided to the job
             :param str fsID: job store file ID
             :param str maxWriteFile: path to file where the max number of concurrent readers of
                                      cache lock file will be written
             """
-            fsID = fileInfo
             work_dir = job.fileStore.getLocalTempDir()
             outfile = job.fileStore.readGlobalFile(fsID, '/'.join([work_dir, 'temp']), cache=True,
                                                    mutable=False)