Repository: databiosphere/toil
Issue: JobStore problem when using GridEngine batch system

Using the latest git version of Toil (last commit: 2ad4bad) with a gridengine batch system results in the following error:

``` bash
> python HelloWorld.py --batchSystem gridEngine --disableCaching --logDebug file:jobStore
pc 2016-08-10 09:15:11,849 MainThread INFO toil.lib.bioio: Logging set at level: DEBUG
pc 2016-08-10 09:15:11,849 MainThread INFO toil.lib.bioio: Logging set at level: DEBUG
pc 2016-08-10 09:15:11,873 MainThread INFO toil.jobStores.fileJobStore: Path to job store directory is '/data/cwl-test/jobStore'.
pc 2016-08-10 09:15:11,901 MainThread INFO toil.jobStores.abstractJobStore: The workflow ID is: '095897de-df05-40d2-99ab-8bb5d1520d44'
pc 2016-08-10 09:15:11,920 MainThread INFO toil.common: Using the gridengine batch system
Traceback (most recent call last):
  File "HelloWorld.py", line 11, in <module>
    print Job.Runner.startToil(j, options) #Prints Hello, world!, ...
  File "/data/cwl-test/toil_git/lib/python2.7/site-packages/toil/job.py", line 542, in startToil
    return toil.start(job)
  File "/data/cwl-test/toil_git/lib/python2.7/site-packages/toil/common.py", line 550, in start
    userScript=rootJob.getUserScript())
  File "/data/cwl-test/toil_git/lib/python2.7/site-packages/toil/common.py", line 708, in createBatchSystem
    return batchSystemClass(**kwargs)
  File "/data/cwl-test/toil_git/lib/python2.7/site-packages/toil/batchSystems/gridengine.py", line 219, in __init__
    self.gridengineResultsFileHandle = open(self.gridengineResultsFile, 'w')
IOError: [Errno 2] No such file or directory: 'file:jobStore/results.txt'
```

Running the script locally (i.e., without `--batchSystem gridEngine --disableCaching`) is working without any errors.

NB: `HelloWorld.py` is taken from here: http://toil.readthedocs.io/en/releases-3.3.x/running.html

Thank you very much for your great work!


Changed files:
src/toil/batchSystems/gridengine.py, src/toil/test/utils/utilsTest.py, src/toil/utils/toilKill.py, src/toil/test/batchSystems/batchSystemTest.py, src/toil/test/mesos/stress.py, src/toil/utils/toilClean.py, src/toil/common.py, src/toil/test/mesos/mesosTest.py, src/toil/utils/toilStatus.py, src/toil/utils/toilStats.py

diff --git a/src/toil/batchSystems/gridengine.py b/src/toil/batchSystems/gridengine.py
index 510a9313..0528acd6 100644
--- a/src/toil/batchSystems/gridengine.py
+++ b/src/toil/batchSystems/gridengine.py
@@ -222,7 +222,9 @@ class GridengineBatchSystem(BatchSystemSupport):
 
     def __init__(self, config, maxCores, maxMemory, maxDisk):
         super(GridengineBatchSystem, self).__init__(config, maxCores, maxMemory, maxDisk)
-        self.gridengineResultsFile = self._getResultsFileName(config.jobStore)
+        prefix = 'file:'
+        assert config.jobStore.startswith(prefix)
+        self.gridengineResultsFile = self._getResultsFileName(config.jobStore[len(prefix):])
         # Reset the job queue and results (initially, we do this again once we've killed the jobs)
         self.gridengineResultsFileHandle = open(self.gridengineResultsFile, 'w')
         # We lose any previous state in this file, and ensure the files existence
diff --git a/src/toil/common.py b/src/toil/common.py
index caf6a4b3..4ef19741 100644
--- a/src/toil/common.py
+++ b/src/toil/common.py
@@ -43,7 +43,7 @@ class Config(object):
         self.jobStore is the same, e.g. when a job store name is reused after a previous run has
         finished sucessfully and its job store has been clean up."""
         self.workflowAttemptNumber = None
-        self.jobStore = os.path.abspath("./toil")
+        self.jobStore = None
         self.logLevel = getLogLevelString()
         self.workDir = None
         self.stats = False
@@ -144,9 +144,17 @@ class Config(object):
                 assert isinstance(maxValue, float)
                 return lambda x: minValue <= x < maxValue
 
+        def parseJobStore(s):
+            name, rest = Toil.parseLocator(s)
+            if name == 'file':
+                # We need to resolve relative paths early, on the leader, because the worker process
+                # may have a different working directory than the leader, e.g. under Mesos.
+                return Toil.buildLocator(name, os.path.abspath(rest))
+            else:
+                return s
+
         #Core options
-        setOption("jobStore",
-                  parsingFn=lambda x: os.path.abspath(x) if options.jobStore.startswith('.') else x)
+        setOption("jobStore", parsingFn=parseJobStore)
         #TODO: LOG LEVEL STRING
         setOption("workDir")
         setOption("stats")
@@ -604,8 +612,8 @@ class Toil(object):
         finally:
             self._shutdownBatchSystem()
 
-    @staticmethod
-    def getJobStore(locator):
+    @classmethod
+    def getJobStore(cls, locator):
         """
         Create an instance of the concrete job store implementation that matches the given locator.
 
@@ -614,27 +622,16 @@ class Toil(object):
         :return: an instance of a concrete subclass of AbstractJobStore
         :rtype: toil.jobStores.abstractJobStore.AbstractJobStore
         """
-        if locator[0] in '/.':
-            locator = 'file:' + locator
-
-        try:
-            name, rest = locator.split(':', 1)
-        except ValueError:
-            raise RuntimeError('Invalid job store locator syntax.')
-
+        name, rest = cls.parseLocator(locator)
         if name == 'file':
             from toil.jobStores.fileJobStore import FileJobStore
             return FileJobStore(rest)
-
         elif name == 'aws':
             from toil.jobStores.aws.jobStore import AWSJobStore
             return AWSJobStore(rest)
-
         elif name == 'azure':
             from toil.jobStores.azureJobStore import AzureJobStore
-            account, namePrefix = rest.split(':', 1)
             return AzureJobStore(rest)
-
         elif name == 'google':
             from toil.jobStores.googleJobStore import GoogleJobStore
             projectID, namePrefix = rest.split(':', 1)
@@ -642,6 +639,23 @@ class Toil(object):
         else:
             raise RuntimeError("Unknown job store implementation '%s'" % name)
 
+    @staticmethod
+    def parseLocator(locator):
+        if locator[0] in '/.' or ':' not in locator:
+            return 'file', locator
+        else:
+            try:
+                name, rest = locator.split(':', 1)
+            except ValueError:
+                raise RuntimeError('Invalid job store locator syntax.')
+            else:
+                return name, rest
+
+    @staticmethod
+    def buildLocator(name, rest):
+        assert ':' not in name
+        return name + ':' + rest
+
     @classmethod
     def resumeJobStore(cls, locator):
         jobStore = cls.getJobStore(locator)
diff --git a/src/toil/test/batchSystems/batchSystemTest.py b/src/toil/test/batchSystems/batchSystemTest.py
index 25257936..8ae8d850 100644
--- a/src/toil/test/batchSystems/batchSystemTest.py
+++ b/src/toil/test/batchSystems/batchSystemTest.py
@@ -546,7 +546,7 @@ class GridEngineBatchSystemTest(hidden.AbstractBatchSystemTest):
     def _createDummyConfig(self):
         config = super(GridEngineBatchSystemTest, self)._createDummyConfig()
         # can't use _getTestJobStorePath since that method removes the directory
-        config.jobStore = self._createTempDir('jobStore')
+        config.jobStore = 'file:'+ self._createTempDir('jobStore')
         return config
 
     def createBatchSystem(self):
diff --git a/src/toil/test/mesos/mesosTest.py b/src/toil/test/mesos/mesosTest.py
index ecb70c93..ff0504e4 100644
--- a/src/toil/test/mesos/mesosTest.py
+++ b/src/toil/test/mesos/mesosTest.py
@@ -60,7 +60,7 @@ class MesosTest(ToilTest, MesosTestSupport):
     def test_hello_world(self):
         system([sys.executable,
                 '-m', helloWorld.__name__,
-                './toilTest',
+                'file:./toilTest',
                 '--batchSystem=mesos',
                 '--logLevel', getLogLevelString()])
 
diff --git a/src/toil/test/mesos/stress.py b/src/toil/test/mesos/stress.py
index 5885c0d6..a8435671 100644
--- a/src/toil/test/mesos/stress.py
+++ b/src/toil/test/mesos/stress.py
@@ -12,6 +12,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 from __future__ import absolute_import
+
 import sys
 from argparse import ArgumentParser
 
@@ -63,7 +64,7 @@ def main(numJobs):
     # Boilerplate -- startToil requires options
     parser = ArgumentParser()
     Job.Runner.addToilOptions(parser)
-    options = parser.parse_args( args=["./toilTest"] )
+    options = parser.parse_args( args=['./toilTest'] )
     options.batchSystem="mesos"
     # Launch first toil Job
     i = LongTestJob( numJobs )
diff --git a/src/toil/test/utils/utilsTest.py b/src/toil/test/utils/utilsTest.py
index 092fbf8e..8e4ed3e8 100644
--- a/src/toil/test/utils/utilsTest.py
+++ b/src/toil/test/utils/utilsTest.py
@@ -27,7 +27,7 @@ from toil.lib.bioio import system
 from toil.test import ToilTest
 from toil.test.sort.sortTest import makeFileToSort
 from toil.utils.toilStats import getStats, processData
-from toil.common import Toil
+from toil.common import Toil, Config
 
 
 class UtilsTest(ToilTest):
@@ -177,7 +177,9 @@ class UtilsTest(ToilTest):
         options.clean = 'never'
         options.stats = True
         Job.Runner.startToil(RunTwoJobsPerWorker(), options)
-        jobStore = Toil.resumeJobStore(options.jobStore)
+        config = Config()
+        config.setOptions(options)
+        jobStore = Toil.resumeJobStore(config.jobStore)
         stats = getStats(jobStore)
         collatedStats = processData(jobStore.config, stats)
         self.assertTrue(len(collatedStats.job_types) == 2,
diff --git a/src/toil/utils/toilClean.py b/src/toil/utils/toilClean.py
index ef44c963..c785d38a 100644
--- a/src/toil/utils/toilClean.py
+++ b/src/toil/utils/toilClean.py
@@ -19,7 +19,7 @@ import logging
 
 from toil.lib.bioio import getBasicOptionParser
 from toil.lib.bioio import parseBasicOptions
-from toil.common import Toil, jobStoreLocatorHelp
+from toil.common import Toil, jobStoreLocatorHelp, Config
 from toil.version import version
 
 logger = logging.getLogger( __name__ )
@@ -29,8 +29,9 @@ def main():
     parser.add_argument("jobStore", type=str,
                         help="The location of the job store to delete. " + jobStoreLocatorHelp)
     parser.add_argument("--version", action='version', version=version)
-    options = parseBasicOptions(parser)
+    config = Config()
+    config.setOptions(parseBasicOptions(parser))
     logger.info("Attempting to delete the job store")
-    jobStore = Toil.getJobStore(options.jobStore)
+    jobStore = Toil.getJobStore(config.jobStore)
     jobStore.destroy()
     logger.info("Successfully deleted the job store")
diff --git a/src/toil/utils/toilKill.py b/src/toil/utils/toilKill.py
index 40e4a674..cadf2444 100644
--- a/src/toil/utils/toilKill.py
+++ b/src/toil/utils/toilKill.py
@@ -19,7 +19,7 @@ import logging
 
 from toil.lib.bioio import getBasicOptionParser
 from toil.lib.bioio import parseBasicOptions
-from toil.common import Toil, jobStoreLocatorHelp
+from toil.common import Toil, jobStoreLocatorHelp, Config
 from toil.version import version
 
 logger = logging.getLogger( __name__ )
@@ -32,10 +32,11 @@ def main():
                              "be killed." + jobStoreLocatorHelp)
     parser.add_argument("--version", action='version', version=version)
     options = parseBasicOptions(parser)
+    config = Config()
+    config.setOptions(options)
+    jobStore = Toil.resumeJobStore(config.jobStore)
 
-    jobStore = Toil.resumeJobStore(options.jobStore)
-
-    logger.info("Starting routine to kill running jobs in the toil workflow: %s" % options.jobStore)
+    logger.info("Starting routine to kill running jobs in the toil workflow: %s", config.jobStore)
     ####This behaviour is now broken
     batchSystem = Toil.createBatchSystem(jobStore.config) #This should automatically kill the existing jobs.. so we're good.
     for jobID in batchSystem.getIssuedBatchJobIDs(): #Just in case we do it again.
diff --git a/src/toil/utils/toilStats.py b/src/toil/utils/toilStats.py
index 3265d7c6..91c5257f 100644
--- a/src/toil/utils/toilStats.py
+++ b/src/toil/utils/toilStats.py
@@ -22,7 +22,7 @@ import logging
 import json
 from toil.lib.bioio import getBasicOptionParser
 from toil.lib.bioio import parseBasicOptions
-from toil.common import Toil, jobStoreLocatorHelp
+from toil.common import Toil, jobStoreLocatorHelp, Config
 from toil.version import version
 from bd2k.util.expando import Expando
 
@@ -597,7 +597,9 @@ def main():
     initializeOptions(parser)
     options = parseBasicOptions(parser)
     checkOptions(options, parser)
-    jobStore = Toil.resumeJobStore(options.jobStore)
+    config = Config()
+    config.setOptions(options)
+    jobStore = Toil.resumeJobStore(config.jobStore)
     stats = getStats(jobStore)
     collatedStatsTag = processData(jobStore.config, stats)
     reportData(collatedStatsTag, options)
diff --git a/src/toil/utils/toilStatus.py b/src/toil/utils/toilStatus.py
index 537229b4..d0791197 100644
--- a/src/toil/utils/toilStatus.py
+++ b/src/toil/utils/toilStatus.py
@@ -24,7 +24,7 @@ import sys
 from toil.lib.bioio import logStream
 from toil.lib.bioio import getBasicOptionParser
 from toil.lib.bioio import parseBasicOptions
-from toil.common import Toil, jobStoreLocatorHelp
+from toil.common import Toil, jobStoreLocatorHelp, Config
 from toil.leader import ToilState
 from toil.job import JobException
 from toil.version import version
@@ -67,12 +67,13 @@ def main():
     
     logger.info("Checking if we have files for Toil")
     assert options.jobStore is not None
-
+    config = Config()
+    config.setOptions(options)
     ##########################################
     #Survey the status of the job and report.
     ##########################################  
     
-    jobStore = Toil.resumeJobStore(options.jobStore)
+    jobStore = Toil.resumeJobStore(config.jobStore)
     try:
         rootJob = jobStore.loadRootJob()
     except JobException:
@@ -90,7 +91,7 @@ def main():
 
     print('There are %i active jobs, %i parent jobs with children, and %i totally failed jobs '
           'currently in %s.' % (len(toilState.updatedJobs), len(toilState.successorCounts),
-                                len(failedJobs), options.jobStore), file=sys.stderr)
+                                len(failedJobs), config.jobStore), file=sys.stderr)
     
     if options.verbose: #Verbose currently means outputting the files that have failed.
         for job in failedJobs: