Repository: DataBiosphere/toil

Issue:
Convenience job attributes
I was talking with a few other pipeline developers and a few additions to the `job` instance, like `job.cores`,  would cut down on boilerplate and make the code more readable.

- [ ] `job.tempDir` —  At the top of every job function I have to write `work_dir = job.fileStore.getLocalTempDir()` to create and get the localTempDir path. Instead, have `job.tempDir` return the file path from `job.fileStore.getLocalTempDir()`, but don't require me to invoke it to create the local temp dir, i.e. create the directory when invoked for the first time in a job function or just always create it if that overhead isn't large. 

- [ ] `job.log` — We could just point this to `job.fileStore.logToMaster` or w/e the preferred logging  system is. 


Discussion:
+1
ill take a crack at it...
@jessebrennan — That's my kinda enthusiasm. 

Resolving patch:
diff --git a/src/toil/job.py b/src/toil/job.py
--- a/src/toil/job.py
+++ b/src/toil/job.py
@@ -305,6 +305,7 @@ def __init__(self, memory=None, cores=None, disk=None, preemptable=None, unitNam
         self._rvs = collections.defaultdict(list)
         self._promiseJobStore = None
         self._fileStore = None
+        self._tempDir = None
 
     def run(self, fileStore):
         """
@@ -457,6 +458,24 @@ def addFollowOnJobFn(self, fn, *args, **kwargs):
         else:
             return self.addFollowOn(JobFunctionWrappingJob(fn, *args, **kwargs))
 
+    @property
+    def tempDir(self):
+        """
+        Shortcut to calling :func:`job.fileStore.getLocalTempDir`. Temp dir is created on first call
+        and will be returned for first and future calls
+        :return: Path to tempDir. See `job.fileStore.getLocalTempDir`
+        :rtype: str
+        """
+        if self._tempDir is None:
+            self._tempDir = self._fileStore.getLocalTempDir()
+        return self._tempDir
+
+    def log(self, text, level=logging.INFO):
+        """
+        convenience wrapper for :func:`fileStore.logToMaster`
+        """
+        self._fileStore.logToMaster(text, level)
+
     @staticmethod
     def wrapFn(fn, *args, **kwargs):
         """


Test patch:
diff --git a/src/toil/test/sort/sort.py b/src/toil/test/sort/sort.py
--- a/src/toil/test/sort/sort.py
+++ b/src/toil/test/sort/sort.py
@@ -37,7 +37,7 @@ def setup(job, inputFile, N, downCheckpoints, options):
     Sets up the sort.
     Returns the FileID of the sorted file
     """
-    job.fileStore.logToMaster("Starting the merge sort")
+    job.log("Starting the merge sort")
     return job.addChildJobFn(down,
                              inputFile, N,
                              downCheckpoints,
@@ -58,8 +58,8 @@ def down(job, inputFileStoreID, N, downCheckpoints, options, memory=sortMemory):
     length = os.path.getsize(inputFile)
     if length > N:
         # We will subdivide the file
-        job.fileStore.logToMaster("Splitting file: %s of size: %s"
-                                  % (inputFileStoreID, length), level=logging.CRITICAL)
+        job.log("Splitting file: %s of size: %s"
+                % (inputFileStoreID, length), level=logging.CRITICAL)
         # Split the file into two copies
         midPoint = getMidPoint(inputFile, 0, length)
         t1 = job.fileStore.getLocalTempFile()
@@ -77,8 +77,8 @@ def down(job, inputFileStoreID, N, downCheckpoints, options, memory=sortMemory):
                                                       checkpoint=downCheckpoints, options=options, memory=options.mergeMemory).rv(), options=options, memory=options.sortMemory).rv()
     else:
         # We can sort this bit of the file
-        job.fileStore.logToMaster("Sorting file: %s of size: %s"
-                                  % (inputFileStoreID, length), level=logging.CRITICAL)
+        job.log("Sorting file: %s of size: %s"
+                % (inputFileStoreID, length), level=logging.CRITICAL)
         # Sort the copy and write back to the fileStore
         shutil.copyfile(inputFile, inputFile + '.sort')
         sort(inputFile + '.sort')
@@ -93,8 +93,8 @@ def up(job, inputFileID1, inputFileID2, options, memory=sortMemory):
         with job.fileStore.readGlobalFileStream(inputFileID1) as inputFileHandle1:
             with job.fileStore.readGlobalFileStream(inputFileID2) as inputFileHandle2:
                 merge(inputFileHandle1, inputFileHandle2, fileHandle)
-                job.fileStore.logToMaster("Merging %s and %s to %s"
-                                          % (inputFileID1, inputFileID2, outputFileStoreID))
+                job.log("Merging %s and %s to %s"
+                        % (inputFileID1, inputFileID2, outputFileStoreID))
         # Cleanup up the input files - these deletes will occur after the completion is successful.
         job.fileStore.deleteGlobalFile(inputFileID1)
         job.fileStore.deleteGlobalFile(inputFileID2)
diff --git a/src/toil/test/src/fileStoreTest.py b/src/toil/test/src/fileStoreTest.py
--- a/src/toil/test/src/fileStoreTest.py
+++ b/src/toil/test/src/fileStoreTest.py
@@ -733,7 +733,7 @@ def testAsyncWriteWithCaching(self):
             A = Job.wrapJobFn(self._forceModifyCacheLockFile, newTotalMB=1024, disk='1G')
             B = Job.wrapJobFn(self._doubleWriteFileToJobStore, fileMB=850, disk='900M')
             # Set it to > 2GB such that the cleanup jobs don't die.
-            C = Job.wrapJobFn(self._readFromJobStoreWithoutAsssertions, fsID=B.rv(), disk='1G')
+            C = Job.wrapJobFn(self._readFromJobStoreWithoutAssertions, fsID=B.rv(), disk='1G')
             D = Job.wrapJobFn(self._forceModifyCacheLockFile, newTotalMB=5000, disk='1G')
             A.addChild(B)
             B.addChild(C)
@@ -763,13 +763,13 @@ def newHarbingerFileRead(self):
 
             job.fileStore.writeGlobalFile(testFile.name)
             fsID = job.fileStore.writeGlobalFile(testFile.name)
-            hidden.AbstractCachingFileStoreTest._readFromJobStoreWithoutAsssertions(job, fsID)
+            hidden.AbstractCachingFileStoreTest._readFromJobStoreWithoutAssertions(job, fsID)
             # Make this take longer so we can test asynchronous writes across jobs/workers.
             job.fileStore.HarbingerFile.read = newHarbingerFileRead
             return job.fileStore.writeGlobalFile(testFile.name)
 
         @staticmethod
-        def _readFromJobStoreWithoutAsssertions(job, fsID):
+        def _readFromJobStoreWithoutAssertions(job, fsID):
             """
             Reads a file from the job store.  That will be all, thank you.
 
diff --git a/src/toil/test/src/jobTest.py b/src/toil/test/src/jobTest.py
--- a/src/toil/test/src/jobTest.py
+++ b/src/toil/test/src/jobTest.py
@@ -323,6 +323,18 @@ def createWorkflow():
 
         self.runNewCheckpointIsLeafVertexTest(createWorkflow)
 
+    def testTempDir(self):
+        """
+        test that job.tempDir works as expected and make use of job.log for logging
+        """
+        message = "I love rachael price"
+
+        options = Job.Runner.getDefaultOptions(self._getTestJobStorePath())
+        with Toil(options) as workflow:
+            j = sillyTestJob(message)
+            j.addChildJobFn(seriousTestJob, message)
+            workflow.start(j)
+
     def runNewCheckpointIsLeafVertexTest(self, createWorkflowFn):
         """
         Test verification that a checkpoint job is a leaf vertex using both
@@ -693,6 +705,44 @@ def child(job):
     pass
 
 
+class sillyTestJob(Job):
+    """
+    all this job does is write a message to a tempFile
+    in the tempDir (which is deleted)
+    """
+    def __init__(self, message):
+        Job.__init__(self)
+        self.message = message
+
+    @staticmethod
+    def sillify(message):
+        """
+        Turns "this serious help message" into "shis serious selp sessage"
+        """
+        return ' '.join(['s' + word if word[0] in 'aeiou' else 's' + word[1:] for word in message.split()])
+
+    def run(self, fileStore):
+        file1 = self.tempDir + 'sillyFile.txt'
+        self.log('first filename is {}'.format(file1))
+        file2 = self.tempDir + 'sillyFile.txt'
+        self.log('second filename is {}'.format(file1))
+        # make sure we get the same thing every time
+        assert file1 == file2
+
+        # write to the tempDir to be sure that everything works
+        with open(file1, 'w') as fd:
+            fd.write(self.sillify(self.message))
+
+
+def seriousTestJob(job, message):
+    # testing job.temDir for functionJobs
+    with open(job.tempDir + 'seriousFile.txt', 'w') as fd:
+        fd.write("The unadulterated message is:")
+        fd.write(message)
+    # and logging
+    job.log("message has been written")
+
+
 def checkRequirements(job):
     # insure default resource requirements are being set correctly
     assert job.cores is not None