Repository: DataBiosphere/toil

Issue:
Add config.workflowId
Workflows and therefore job stores should have their own unique ID such that we can distinquish between jobstore names that have been resumed.

Add config.workflowId
Workflows and therefore job stores should have their own unique ID such that we can distinquish between jobstore names that have been resumed.



Discussion:

Resolving patch:
diff --git a/src/toil/common.py b/src/toil/common.py
--- a/src/toil/common.py
+++ b/src/toil/common.py
@@ -35,7 +35,12 @@ class Config(object):
     Class to represent configuration operations for a toil workflow run. 
     """
     def __init__(self):
-        #Core options
+        # Core options
+        self.workflowID = None
+        """This attribute uniquely identifies the job store and therefore the workflow. It is
+        necessary in order to distinguish between two consequitive workflows for which
+        self.jobStore is the same, e.g. when a job store name is reused after a previous run has
+        finished sucessfully and its job store has been clean up."""
         self.jobStore = os.path.abspath("./toil")
         self.logLevel = getLogLevelString()
         self.workDir = None
diff --git a/src/toil/jobStores/abstractJobStore.py b/src/toil/jobStores/abstractJobStore.py
--- a/src/toil/jobStores/abstractJobStore.py
+++ b/src/toil/jobStores/abstractJobStore.py
@@ -17,6 +17,7 @@
 from abc import ABCMeta, abstractmethod
 from contextlib import contextmanager
 from datetime import timedelta
+from uuid import uuid4
 
 try:
     import cPickle
@@ -61,18 +62,21 @@ class AbstractJobStore(object):
 
     def __init__(self, config=None):
         """
-        :param config: If config is not None then the \
-        given configuration object will be written to the shared file "config.pickle" which can \
-        later be retrieved using the readSharedFileStream. See writeConfigToStore. \
-        If this file already exists it will be overwritten. If config is None, \
-        the shared file "config.pickle" is assumed to exist and is retrieved. See loadConfigFromStore.
-
+        :param config: If config is not None then the given configuration object will be written
+               to the shared file "config.pickle" which can later be retrieved using the
+               readSharedFileStream. See writeConfigToStore. If this file already exists it will be
+               overwritten. If config is None, the shared file "config.pickle" is assumed to exist
+               and is retrieved. See loadConfigFromStore.
         """
         # Now get on with reading or writing the config
         if config is None:
             with self.readSharedFileStream("config.pickle") as fileHandle:
-                self.__config = cPickle.load(fileHandle)
+                config = cPickle.load(fileHandle)
+                assert config.workflowID is not None
+                self.__config = config
         else:
+            assert config.workflowID is None
+            config.workflowID = str(uuid4())
             self.__config = config
             self.writeConfigToStore()
 
@@ -92,14 +96,14 @@ def config(self):
     def _checkJobStoreCreation(create, exists, jobStoreString):
         """
         Consistency checks which will result in exceptions if we attempt to overwrite an existing
-        jobStore.
+        jobStore. This method must be called by the constructor of a subclass before any
+        modification are made.
 
-        :type create: boolean
+        :type create: bool
 
-        :type exists: boolean
+        :type exists: bool
 
-        :raise JobStoreCreationException:  Thrown if create=True and exists=True or create=False
-                                           and exists=False
+        :raise JobStoreCreationException:  if create == exists
         """
         if create and exists:
             raise JobStoreCreationException("The job store '%s' already exists. "
@@ -150,13 +154,13 @@ def getJob(jobId):
                 return jobCache[jobId]
             else:
                 return self.load(jobId)
-                
+
         def haveJob(jobId):
             if jobCache is not None:
                 return jobCache.has_key(jobId)
             else:
                 return self.exists(jobId)
-                
+
         def getJobs():
             if jobCache is not None:
                 return jobCache.itervalues()
@@ -231,7 +235,7 @@ def getConnectedJobs(jobWrapper):
         # Remove any crufty stats/logging files from the previous run
         logger.info("Discarding old statistics and logs...")
         self.readStatsAndLogging(lambda x: None)
-        
+
         logger.info("Job store is clean")
 
     ##########################################
diff --git a/src/toil/jobStores/fileJobStore.py b/src/toil/jobStores/fileJobStore.py
--- a/src/toil/jobStores/fileJobStore.py
+++ b/src/toil/jobStores/fileJobStore.py
@@ -30,8 +30,9 @@
 logger = logging.getLogger( __name__ )
 
 class FileJobStore(AbstractJobStore):
-    """Represents the toil using a network file system. For doc-strings
-    of functions see AbstractJobStore.
+    """
+    Represents the toil using a network file system. For doc-strings of functions see
+    AbstractJobStore.
     """
 
     def __init__(self, jobStoreDir, config=None):
@@ -41,21 +42,23 @@ def __init__(self, jobStoreDir, config=None):
         :raise RuntimeError: if config != None and the jobStore already exists or
         config == None and the jobStore does not already exists. 
         """
-        #This is root directory in which everything in the store is kept
+        # This is root directory in which everything in the store is kept
         self.jobStoreDir = absSymPath(jobStoreDir)
         logger.info("Jobstore directory is: %s", self.jobStoreDir)
-        #Safety checks for existing jobStore
-        self._checkJobStoreCreation(config != None, os.path.exists(self.jobStoreDir), self.jobStoreDir)
-        #Directory where temporary files go
+        # Safety checks for existing jobStore
+        self._checkJobStoreCreation(create=config is not None,
+                                    exists=os.path.exists(self.jobStoreDir),
+                                    jobStoreString=self.jobStoreDir)
+        # Directory where temporary files go
         self.tempFilesDir = os.path.join(self.jobStoreDir, "tmp")
-        #Creation of jobStore, if necessary
-        if config != None:
+        # Creation of jobStore, if necessary
+        if config is not None:
             os.mkdir(self.jobStoreDir)
             os.mkdir(self.tempFilesDir)
-        #Parameters for creating temporary files
+        # Parameters for creating temporary files
         self.validDirs = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789"
         self.levels = 2
-        super( FileJobStore, self ).__init__( config=config )
+        super(FileJobStore, self).__init__(config=config)
 
     def deleteJobStore(self):
         if os.path.exists(self.jobStoreDir):


Test patch: