Repository: DataBiosphere/toil

Issue:
Remove the use of getcwd() for temporary files in tests



Discussion:
Put these directories/files in a test build directory

... and use toil.test.ToilTest#_getTestJobStorePath to do so, or another method in a base class

Assigning this to Hannes, as I won't be able to complete this.


Resolving patch:
diff --git a/src/toil/common.py b/src/toil/common.py
--- a/src/toil/common.py
+++ b/src/toil/common.py
@@ -50,7 +50,10 @@ def __init__(self):
         self.logLevel = getLogLevelString()
         self.workDir = None
         self.stats = False
-        self.clean = "never"
+
+        # Because the stats option needs the jobStore to persist past the end of the run,
+        # the clean default value depends the specified stats option and is determined in setOptions
+        self.clean = None
         
         #Restarting the workflow options
         self.restart = False
@@ -115,11 +118,14 @@ def f(x):
         setOption("stats")
         setOption("clean")
         if self.stats:
-            if self.clean != "never":
+            if self.clean != "never" and self.clean is not None:
                 raise RuntimeError("Contradicting options passed: Clean flag is set to %s "
                                    "despite the stats flag requiring "
                                    "the jobStore to be intact at the end of the run. " 
                                    "Set clean to \'never\'" % self.clean)
+            self.clean = "never"
+        elif self.clean is None:
+            self.clean = "onSuccess"
         
         #Restarting the workflow options
         setOption("restart") 
@@ -175,7 +181,7 @@ def _addOptions(addGroupFn, config):
                             "Choices: 'always', 'onError','never', 'onSuccess'. The --stats option requires "
                             "information from the jobStore upon completion so the jobStore will never be deleted with"
                             "that flag. If you wish to be able to restart the run, choose \'never\' or \'onSuccess\'. "
-                            "Default is never if stats collection is enabled, and onSuccess otherwise"))
+                            "Default is \'never\' if stats is enabled, and \'onSuccess\' otherwise"))
 
     #
     #Restarting the workflow options
diff --git a/src/toil/utils/toilClean.py b/src/toil/utils/toilClean.py
new file mode 100644
--- /dev/null
+++ b/src/toil/utils/toilClean.py
@@ -0,0 +1,72 @@
+# Copyright (C) 2015 UCSC Computational Genomics Lab
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""Removes the JobStore from a toil run.
+"""
+from __future__ import absolute_import
+import logging
+import os
+import sys
+
+from toil.lib.bioio import getBasicOptionParser
+from toil.lib.bioio import parseBasicOptions
+from toil.common import loadJobStore
+from toil.jobStores.abstractJobStore import JobStoreCreationException
+
+logger = logging.getLogger( __name__ )
+
+def main():
+    """Removes the JobStore from a toil run.
+    """
+
+    ##########################################
+    #Construct the arguments.
+    ##########################################
+
+    parser = getBasicOptionParser("usage: %prog [--jobStore] JOB_STORE", "%prog 0.1")
+
+    parser.add_option("--jobStore", dest="jobStore",
+                      help="Job store path. Can also be specified as the single argument to the script.\
+                       default=%default", default=os.path.abspath("./toil"))
+
+    options, args = parseBasicOptions(parser)
+    logger.info("Parsed arguments")
+
+    # the JobStore may also be passed as an argument directly to the script
+    assert len(args) <= 1
+    if len(args) == 1:
+        options.jobStore = args[0]
+
+    ##########################################
+    #Do some checks.
+    ##########################################
+
+    logger.info("Checking if we have files for toil")
+    assert options.jobStore != None
+
+    ##########################################
+    #Survey the status of the job and report.
+    ##########################################
+    try:
+        jobStore = loadJobStore(options.jobStore)
+    except JobStoreCreationException:
+        logger.info("The specified JobStore does not exist, it may have already been deleted")
+        sys.exit(0)
+
+    logger.info("Deleting the JobStore")
+    jobStore.deleteJobStore()
+
+
+def _test():
+    import doctest
+    return doctest.testmod()
diff --git a/src/toil/utils/toilMain.py b/src/toil/utils/toilMain.py
--- a/src/toil/utils/toilMain.py
+++ b/src/toil/utils/toilMain.py
@@ -27,7 +27,7 @@ def main():
 
 def loadModules():
     # noinspection PyUnresolvedReferences
-    from toil.utils import toilKill, toilRestart, toilStats, toilStatus
+    from toil.utils import toilKill, toilRestart, toilStats, toilStatus, toilClean
     return {name[4:].lower(): module for name, module in locals().iteritems()}
 
 


Test patch:
diff --git a/src/toil/test/sort/sortTest.py b/src/toil/test/sort/sortTest.py
--- a/src/toil/test/sort/sortTest.py
+++ b/src/toil/test/sort/sortTest.py
@@ -14,7 +14,8 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-"""Tests the scriptTree toil-script compiler.
+"""
+Tests the scriptTree toil-script compiler.
 """
 
 from __future__ import absolute_import
@@ -23,8 +24,7 @@
 import random
 from uuid import uuid4
 import logging
-import shutil
-import tempfile
+import subprocess
 
 from toil.job import Job, JobException
 from toil.lib.bioio import getLogLevelString
@@ -41,16 +41,7 @@
 class SortTest(ToilTest, MesosTestSupport):
     def setUp(self):
         super(SortTest, self).setUp()
-        self.jobStore = self._getTestJobStorePath()
-        self.tempDir = tempfile.mkdtemp(prefix="tempDir")
-        self.testNo = 5
-
-    def tearDown(self):
-        super(SortTest, self).tearDown()
-        if os.path.exists(self.jobStore):
-            shutil.rmtree(self.jobStore)
-        if os.path.exists(self.tempDir):
-            shutil.rmtree(self.tempDir)
+        self.tempDir = self._createTempDir(purpose='tempDir')
 
     def toilSortTest(self, jobStore, batchSystem, lines, N, testNo=1, maxLineLength=10):
         """
@@ -58,73 +49,77 @@ def toilSortTest(self, jobStore, batchSystem, lines, N, testNo=1, maxLineLength=
         """
         for test in xrange(testNo):
 
-            # Specify options
-            options = Job.Runner.getDefaultOptions()
-            options.jobStore = jobStore
-            options.logLevel = getLogLevelString()
-            options.retryCount = 2
-            options.batchSystem = batchSystem
-
-            # Make the file to sort
-            tempSortFile = os.path.join(self.tempDir, "fileToSort.txt")
-            makeFileToSort(tempSortFile, lines=lines, maxLineLength=maxLineLength)
-
-            # First make our own sorted version
-            with open(tempSortFile, 'r') as fileHandle:
-                l = fileHandle.readlines()
-                l.sort()
-
-            # Make the first job
-            firstJob = Job.wrapJobFn(setup, tempSortFile, N, memory=5000)
-
-            # Check we get an exception if we try to restart a workflow that doesn't exist
-            options.restart = True
-            try:
-                Job.Runner.startToil(firstJob, options)
-                self.fail()
-            except JobStoreCreationException:
-                pass
-
-            options.restart = False
-
-            # Now actually run the workflow
-            try:
-                Job.Runner.startToil(firstJob, options)
-                i = 0
-            except FailedJobsException as e:
-                i = e.numberOfFailedJobs
-
-            # Check we get an exception if we try to run without restart on an existing job store
             try:
-                Job.Runner.startToil(firstJob, options)
-                self.fail()
-            except JobStoreCreationException:
-                pass
+                # Specify options
+                options = Job.Runner.getDefaultOptions()
+                options.jobStore = jobStore
+                options.logLevel = getLogLevelString()
+                options.retryCount = 2
+                options.batchSystem = batchSystem
+                options.clean = "never"
+
+                # Make the file to sort
+                tempSortFile = os.path.join(self.tempDir, "fileToSort.txt")
+                makeFileToSort(tempSortFile, lines=lines, maxLineLength=maxLineLength)
+
+                # First make our own sorted version
+                with open(tempSortFile, 'r') as fileHandle:
+                    l = fileHandle.readlines()
+                    l.sort()
+
+                # Make the first job
+                firstJob = Job.wrapJobFn(setup, tempSortFile, N, memory=5000)
+
+                # Check we get an exception if we try to restart a workflow that doesn't exist
+                options.restart = True
+                try:
+                    Job.Runner.startToil(firstJob, options)
+                    self.fail()
+                except JobStoreCreationException:
+                    pass
 
-            options.restart = True
+                options.restart = False
 
-            # This loop tests the restart behavior
-            while i != 0:
-                options.useExistingOptions = random.random() > 0.5
+                # Now actually run the workflow
                 try:
                     Job.Runner.startToil(firstJob, options)
                     i = 0
                 except FailedJobsException as e:
                     i = e.numberOfFailedJobs
 
-            # Now check that if you try to restart from here it will raise an exception
-            # indicating that there are no jobs remaining in the workflow.
-            try:
-                Job.Runner.startToil(firstJob, options)
-                self.fail()
-            except JobException:
-                pass
-                # self.assertTrue(e.message.endswith('left in toil workflow (workflow has finished successfully?)'))
-
-            # Now check the file is properly sorted..
-            with open(tempSortFile, 'r') as fileHandle:
-                l2 = fileHandle.readlines()
-                checkEqual(l, l2)
+                # Check we get an exception if we try to run without restart on an existing store
+                try:
+                    Job.Runner.startToil(firstJob, options)
+                    self.fail()
+                except JobStoreCreationException:
+                    pass
+
+                options.restart = True
+
+                # This loop tests the restart behavior
+                while i != 0:
+                    options.useExistingOptions = random.random() > 0.5
+                    try:
+                        Job.Runner.startToil(firstJob, options)
+                        i = 0
+                    except FailedJobsException as e:
+                        i = e.numberOfFailedJobs
+
+                # Now check that if you try to restart from here it will raise an exception
+                # indicating that there are no jobs remaining in the workflow.
+                try:
+                    Job.Runner.startToil(firstJob, options)
+                    self.fail()
+                except JobException:
+                    pass
+
+                # Now check the file is properly sorted..
+                with open(tempSortFile, 'r') as fileHandle:
+                    l2 = fileHandle.readlines()
+                    checkEqual(l, l2)
+            finally:
+                subprocess.check_call([self._getUtilScriptPath('toilMain'), 'clean',
+                                       '--jobStore', jobStore])
 
     @needs_aws
     def testToilSortOnAWS(self):
@@ -165,6 +160,8 @@ def testToilSort(self):
 
     # The following functions test the functions in the test!
 
+    testNo = 5
+
     def testSort(self):
         for test in xrange(self.testNo):
             tempFile1 = os.path.join(self.tempDir, "fileToSort.txt")
diff --git a/src/toil/test/utils/utilsTest.py b/src/toil/test/utils/utilsTest.py
--- a/src/toil/test/utils/utilsTest.py
+++ b/src/toil/test/utils/utilsTest.py
@@ -116,6 +116,11 @@ def testUtilsSort(self):
             l2 = fileHandle.readlines()
             checkEqual(self.correctSort, l2)
 
+        # Check we can run 'toil clean'
+        toilCleanString = ("{self.toilMain} clean "
+                           "--jobStore {self.toilDir}".format(**locals()))
+        system(toilCleanString)
+            
     def testUtilsStatsSort(self):
         """
         Tests the stats commands on a complete run of the stats test.