Repository: DataBiosphere/toil

Issue:
Reconsider cache ejection strategy
If cache is full, throwing out the smallest items first might not be the best strategy. Maybe removing least recent will be better? Argument for latter

```
def align(job, fastqs):
    '''
    DOCSTRING
    '''
    samfile = align_dna(fastqs)
    samfile = writeGlobalFile(samfile)
    return samfile

def sam_2_bam(job, samfile):
    '''
    DOCSTRING
    '''
    bamfile = job.readGlobalFile(bamfile)
    bamfile = samtools_view(samfile)
    bamfile = writeGlobalFile(bamfile)
    return samfile


def bamsort(job, bamfile):
    '''
    DOCSTRING
    '''
    sorted_bamfile = job.readGlobalFile(sorted_bamfile)
    sorted_bamfile = samtools_view(bamfile)
    sorted_bamfile = writeGlobalFile(sorted_bamfile)
    return sorted_bamfile



def mutation_calling(job, sorted_bamfile):
    '''
    DOCSTRING
    '''
    mutations = job.readGlobalFile(mutations)
    mutations = samtools_view(bamfile)





A = Job.wrapJobFn(align, fastqs)
B = Job.wrapJobFn(sam_2_bam. A.rv())
C = Job.wrapJobFn(bamsort, B.rv())
D = Job.wrapJobFn(mutation_calling, C.rv())
A.addChild(B)
B.addChild(C)
C.addChild(D)
Job.Runner.startToil(A, params)
```

of the three files, samfile > bamfile > sorted_bamfile in terms of size. If cacheSize < (samfile + bamfile), bamfile will be removed and not samfile and then you get a similar situation to issue #489 when C runs.

Reconsider cache ejection strategy
If cache is full, throwing out the smallest items first might not be the best strategy. Maybe removing least recent will be better? Argument for latter

```
def align(job, fastqs):
    '''
    DOCSTRING
    '''
    samfile = align_dna(fastqs)
    samfile = writeGlobalFile(samfile)
    return samfile

def sam_2_bam(job, samfile):
    '''
    DOCSTRING
    '''
    bamfile = job.readGlobalFile(bamfile)
    bamfile = samtools_view(samfile)
    bamfile = writeGlobalFile(bamfile)
    return samfile


def bamsort(job, bamfile):
    '''
    DOCSTRING
    '''
    sorted_bamfile = job.readGlobalFile(sorted_bamfile)
    sorted_bamfile = samtools_view(bamfile)
    sorted_bamfile = writeGlobalFile(sorted_bamfile)
    return sorted_bamfile



def mutation_calling(job, sorted_bamfile):
    '''
    DOCSTRING
    '''
    mutations = job.readGlobalFile(mutations)
    mutations = samtools_view(bamfile)





A = Job.wrapJobFn(align, fastqs)
B = Job.wrapJobFn(sam_2_bam. A.rv())
C = Job.wrapJobFn(bamsort, B.rv())
D = Job.wrapJobFn(mutation_calling, C.rv())
A.addChild(B)
B.addChild(C)
C.addChild(D)
Job.Runner.startToil(A, params)
```

of the three files, samfile > bamfile > sorted_bamfile in terms of size. If cacheSize < (samfile + bamfile), bamfile will be removed and not samfile and then you get a similar situation to issue #489 when C runs.



Discussion:
Also related to #486

Yes, better. Feel free to take this one on. Should be a one-liner!

On Mon, Oct 12, 2015 at 5:57 PM, Arjun Arkal Rao notifications@github.com
wrote:

> If cache is full, throwing out the smallest items first might not be the
> best strategy. Maybe removing least recent will be better? Argument for
> latter
> 
> def align(job, fastqs):
>     '''
>     DOCSTRING
>     '''
>     samfile = align_dna(fastqs)
>     samfile = writeGlobalFile(samfile)
>     return samfile
> 
> def sam_2_bam(job, samfile):
>     '''
>     DOCSTRING
>     '''
>     bamfile = job.readGlobalFile(bamfile)
>     bamfile = samtools_view(samfile)
>     bamfile = writeGlobalFile(bamfile)
>     return samfile
> 
> def bamsort(job, bamfile):
>     '''
>     DOCSTRING
>     '''
>     sorted_bamfile = job.readGlobalFile(sorted_bamfile)
>     sorted_bamfile = samtools_view(bamfile)
>     sorted_bamfile = writeGlobalFile(sorted_bamfile)
>     return sorted_bamfile
> 
> def mutation_calling(job, sorted_bamfile):
>     '''
>     DOCSTRING
>     '''
>     mutations = job.readGlobalFile(mutations)
>     mutations = samtools_view(bamfile)
> 
> A = Job.wrapJobFn(align, fastqs)
> B = Job.wrapJobFn(sam_2_bam. A.rv())
> C = Job.wrapJobFn(bamsort, B.rv())
> D = Job.wrapJobFn(mutation_calling, C.rv())
> A.addChild(B)
> B.addChild(C)
> C.addChild(D)
> Job.Runner.startToil(A, params)
> 
> of the three files, samfile > bamfile > sorted_bamfile in terms of size.
> If cacheSize < (samfile + bamfile), bamfile will be removed and not samfile
> and then you get a similar situation to issue #489
> https://github.com/BD2KGenomics/toil/issues/489
> 
> —
> Reply to this email directly or view it on GitHub
> https://github.com/BD2KGenomics/toil/issues/490.

Will do

Also related to #486

Yes, better. Feel free to take this one on. Should be a one-liner!

On Mon, Oct 12, 2015 at 5:57 PM, Arjun Arkal Rao notifications@github.com
wrote:

> If cache is full, throwing out the smallest items first might not be the
> best strategy. Maybe removing least recent will be better? Argument for
> latter
> 
> def align(job, fastqs):
>     '''
>     DOCSTRING
>     '''
>     samfile = align_dna(fastqs)
>     samfile = writeGlobalFile(samfile)
>     return samfile
> 
> def sam_2_bam(job, samfile):
>     '''
>     DOCSTRING
>     '''
>     bamfile = job.readGlobalFile(bamfile)
>     bamfile = samtools_view(samfile)
>     bamfile = writeGlobalFile(bamfile)
>     return samfile
> 
> def bamsort(job, bamfile):
>     '''
>     DOCSTRING
>     '''
>     sorted_bamfile = job.readGlobalFile(sorted_bamfile)
>     sorted_bamfile = samtools_view(bamfile)
>     sorted_bamfile = writeGlobalFile(sorted_bamfile)
>     return sorted_bamfile
> 
> def mutation_calling(job, sorted_bamfile):
>     '''
>     DOCSTRING
>     '''
>     mutations = job.readGlobalFile(mutations)
>     mutations = samtools_view(bamfile)
> 
> A = Job.wrapJobFn(align, fastqs)
> B = Job.wrapJobFn(sam_2_bam. A.rv())
> C = Job.wrapJobFn(bamsort, B.rv())
> D = Job.wrapJobFn(mutation_calling, C.rv())
> A.addChild(B)
> B.addChild(C)
> C.addChild(D)
> Job.Runner.startToil(A, params)
> 
> of the three files, samfile > bamfile > sorted_bamfile in terms of size.
> If cacheSize < (samfile + bamfile), bamfile will be removed and not samfile
> and then you get a similar situation to issue #489
> https://github.com/BD2KGenomics/toil/issues/489
> 
> —
> Reply to this email directly or view it on GitHub
> https://github.com/BD2KGenomics/toil/issues/490.

Will do


Resolving patch:
diff --git a/src/toil/job.py b/src/toil/job.py
--- a/src/toil/job.py
+++ b/src/toil/job.py
@@ -442,7 +442,7 @@ def getLocalTempFile(self):
             This file will exist for the duration of the job only, and
             is guaranteed to be deleted once the job terminates.
             """
-            handle, tmpFile = tempfile.mkstemp(prefix="tmp", 
+            handle, tmpFile = tempfile.mkstemp(prefix="tmp",
                                                suffix=".tmp", dir=self.localTempDir)
             os.close(handle)
             return os.path.abspath(tmpFile) 
@@ -667,18 +667,20 @@ def _cleanLocalTempDir(self, cacheSize):
             """
             #Remove files so that the total cached files are smaller than a cacheSize
             
-            #List of pairs of (fileSize, fileStoreID) for cached files
-            cachedFileSizes = map(lambda x : (os.stat(self.jobStoreFileIDToCacheLocation[x]).st_size, x), 
-                                  self.jobStoreFileIDToCacheLocation.keys())
+            #List of pairs of (fileCreateTime, fileStoreID) for cached files
+            cachedFileCreateTimes = map(lambda x : (os.stat(self.jobStoreFileIDToCacheLocation[x]).st_ctime, x),
+                                        self.jobStoreFileIDToCacheLocation.keys())
             #Total number of bytes stored in cached files
-            totalCachedFileSizes = sum(map(lambda x : x[0], cachedFileSizes))
-            #Remove smallest files first - this is not obviously best, could do it a different
-            #way
-            cachedFileSizes.sort()
-            cachedFileSizes.reverse()
+            totalCachedFileSizes = sum([os.stat(self.jobStoreFileIDToCacheLocation[x]).st_size for x in
+                                        self.jobStoreFileIDToCacheLocation.keys()])
+            #Remove earliest created files first - this is in place of 'Remove smallest files first'.  Again, might
+            #not be the best strategy.
+            cachedFileCreateTimes.sort()
+            cachedFileCreateTimes.reverse()
             #Now do the actual file removal
             while totalCachedFileSizes > cacheSize:
-                fileSize, fileStoreID =  cachedFileSizes.pop()
+                fileCreateTime, fileStoreID =  cachedFileCreateTimes.pop()
+                fileSize = os.stat(self.jobStoreFileIDToCacheLocation[fileStoreID]).st_size
                 filePath = self.jobStoreFileIDToCacheLocation[fileStoreID]
                 self.jobStoreFileIDToCacheLocation.pop(fileStoreID)
                 os.remove(filePath)


Test patch:
diff --git a/src/toil/test/src/jobCacheEjectionTest.py b/src/toil/test/src/jobCacheEjectionTest.py
new file mode 100644
--- /dev/null
+++ b/src/toil/test/src/jobCacheEjectionTest.py
@@ -0,0 +1,92 @@
+# Copyright (C) 2015 UCSC Computational Genomics Lab
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+from __future__ import absolute_import, print_function
+import random
+import os
+import tempfile
+
+from toil.job import Job
+from toil.test import ToilTest
+
+class JobCacheEjectionTest(ToilTest):
+    """
+    Tests testing the Job.FileStore class
+    """
+    def testCacheEjection(self):
+        """
+        Test cache always always ejects least recently created file
+        """
+        # Makes three jobs that create an output file each which they write to filestore.  The combined size of any two
+        # files is always less that cacheSize but the combined size of all 3 is always more so 1 file always has to be
+        # ejected. Test to ensure that A is always ejected regardless of size.
+        #  Make a temp directory for the test
+        test_dir = self._createTempDir()
+        for test in xrange(10):
+            options = Job.Runner.getDefaultOptions(self._getTestJobStorePath())
+            options.logLevel = "DEBUG"
+            options.cacheSize = 100000
+            options.retryCount=100
+            options.badWorker=0.5
+            options.badWorkerFailInterval = 1.0
+            # Create a temp file to write teh test results
+            handle, logfile = tempfile.mkstemp(dir=test_dir)
+            os.close(handle)
+            file_sizes = [50000, 40000, 30000]
+            # Randomize to (potentially) test all combinations
+            random.shuffle(file_sizes)
+            # Run the workflow. A, B and C do teh cache operations, and D prints test status to tempFile
+            A = Job.wrapJobFn(fileTestJob, file_sizes[0])
+            B = Job.wrapJobFn(fileTestJob, file_sizes[0])
+            C = Job.wrapJobFn(fileTestJob, file_sizes[0])
+            D = Job.wrapJobFn(fileTestCache, A.rv(), B.rv(), C.rv(), logfile)
+            A.addChild(B)
+            B.addChild(C)
+            C.addChild(D)
+            Job.Runner.startToil(A, options)
+            #  Assert jobs passed by reading test results from tempFile
+            with open(logfile, 'r') as outfile:
+                for test_status in outfile:
+                    assert test_status.strip() == 'True'
+
+
+
+def fileTestCache(job, file_a, file_b, file_c, logfile):
+    """
+    Test job exercises Job.FileStore functions
+    """
+    with open(logfile, 'w') as lf:
+        print(file_a not in job.fileStore.jobStoreFileIDToCacheLocation.items(), file=lf)
+        print(file_b not in job.fileStore.jobStoreFileIDToCacheLocation.items(), file=lf)
+        print(file_c not in job.fileStore.jobStoreFileIDToCacheLocation.items(), file=lf)
+
+def fileTestJob(job, file_size):
+    """
+    Test job exercises Job.FileStore functions
+    """
+    def randomString(str_len):
+        chars = "ABCDEFGHIJKLMNOPQRSTUVWXYZ"
+        s = "".join(map(lambda i : random.choice(chars), xrange(str_len)))
+        return s
+    if random.random() > 0.5:
+        #Make a local copy of the file
+        tempFile = job.fileStore.getLocalTempFile()
+    else:
+        tempDir = job.fileStore.getLocalTempDir()
+        handle, tempFile = tempfile.mkstemp(dir=tempDir)
+        os.close(handle)
+    with open(tempFile, 'w') as fH:
+        fH.write(os.urandom(file_size))
+    job.fileStore.writeGlobalFile(tempFile)
+    return fH.name
+