Repository: DataBiosphere/toil

Issue:
Job.Service needs access to the underlying job instance
We're running into this in https://github.com/BD2KGenomics/toil-lib/pull/46. Specifically, if the `Job.Service.start` method invokes a Docker container via [docker_call](https://github.com/BD2KGenomics/toil-lib/blob/master/src/toil_lib/programs.py#L20) that cleans up the ghost container via a [`defer`red method](https://github.com/BD2KGenomics/toil/blob/master/src/toil/job.py#L549), we need the wrapping `ServiceJob`. To do this, we can swap out the `fileStore` parameter that is passed in the `start`/`stop` methods with the underlying job (since the fileStore can be accessed through the job).



Discussion:
+1. Can you add a service job test  in toil?


Resolving patch:
diff --git a/src/toil/job.py b/src/toil/job.py
--- a/src/toil/job.py
+++ b/src/toil/job.py
@@ -656,11 +656,12 @@ def __init__(self, memory=None, cores=None, disk=None, preemptable=None):
             self._hasParent = False
 
         @abstractmethod
-        def start(self, fileStore):
+        def start(self, job):
             """
             Start the service.
             
-            :param toil.job.Job.FileStore fileStore: A fileStore object to create temporary files with.
+            :param toil.job.Job job: The underlying job that is being run. Can be used to register
+            deferred functions, or to access the fileStore for creating temporary files.
 
             :returns: An object describing how to access the service. The object must be pickleable \
             and will be used by jobs to access the service (see :func:`toil.job.Job.addService`).
@@ -668,12 +669,12 @@ def start(self, fileStore):
             pass
 
         @abstractmethod
-        def stop(self, fileStore):
+        def stop(self, job):
             """
-            Stops the service. 
+            Stops the service. Function can block until complete.
             
-            :param toil.job.Job.FileStore fileStore: A fileStore object to create temporary files with.
-                Function can block until complete.
+            :param toil.job.Job job: The underlying job that is being run. Can be used to register
+            deferred functions, or to access the fileStore for creating temporary files.
             """
             pass
 
@@ -1402,12 +1403,16 @@ def __init__(self, service):
         self.jobWrapper = None
 
     def run(self, fileStore):
+        
+        # we need access to the filestore from underneath the service job
+        self.fileStore = fileStore
+
         # Unpickle the service
         logger.debug('Loading service module %s.', self.serviceModule)
         userModule = self._loadUserModule(self.serviceModule)
         service = self._unpickle( userModule, BytesIO( self.pickledService ), fileStore.jobStore.config )
         #Start the service
-        startCredentials = service.start(fileStore)
+        startCredentials = service.start(self)
         try:
             #The start credentials  must be communicated to processes connecting to
             #the service, to do this while the run method is running we
@@ -1445,16 +1450,13 @@ def run(self, fileStore):
 
                 time.sleep(fileStore.jobStore.config.servicePollingInterval) #Avoid excessive polling
 
-            #Now kill the service
-            #service.stop(fileStore)
-
             # Remove link to the jobWrapper
             self.jobWrapper = None
 
             logger.debug("Service is done")
         finally:
             # The stop function is always called
-            service.stop(fileStore)
+            service.stop(self)
 
     def _run(self, jobWrapper, fileStore):
         # Set the jobWrapper for the job


Test patch:
diff --git a/src/toil/test/src/jobServiceTest.py b/src/toil/test/src/jobServiceTest.py
--- a/src/toil/test/src/jobServiceTest.py
+++ b/src/toil/test/src/jobServiceTest.py
@@ -200,22 +200,22 @@ def __init__(self, messageInt, *args, **kwargs):
         Job.Service.__init__(self, *args, **kwargs)
         self.messageInt = messageInt
 
-    def start(self, fileStore):
+    def start(self, job):
         assert self.disk is not None
         assert self.memory is not None
         assert self.cores is not None
         self.terminate = Event()
         self.error = Event()
-        inJobStoreID = fileStore.jobStore.getEmptyFileStoreID()
-        outJobStoreID = fileStore.jobStore.getEmptyFileStoreID()
+        inJobStoreID = job.fileStore.jobStore.getEmptyFileStoreID()
+        outJobStoreID = job.fileStore.jobStore.getEmptyFileStoreID()
         self.serviceThread = Thread(target=self.serviceWorker,
-                                    args=(fileStore.jobStore, self.terminate, self.error,
+                                    args=(job.fileStore.jobStore, self.terminate, self.error,
                                           inJobStoreID, outJobStoreID,
                                           self.messageInt))
         self.serviceThread.start()
         return (inJobStoreID, outJobStoreID)
 
-    def stop(self, fileStore):
+    def stop(self, job):
         self.terminate.set()
         self.serviceThread.join()
 
@@ -301,10 +301,10 @@ def __init__(self, messageInt, *args, **kwargs):
         Job.Service.__init__(self, *args, **kwargs)
         self.messageInt = messageInt
 
-    def start(self, fileStore):
+    def start(self, job):
         return self.messageInt
 
-    def stop(self, fileStore):
+    def stop(self, job):
         pass
 
     def check(self):