Repository: DataBiosphere/toil

Issue:
Add type hints to serviceManager.py
Adds type hints to src/toil/serviceManager.py so it can be checked under mypy during linting.

Refers to #3568.

┆Issue is synchronized with this [Jira Task](https://ucsc-cgl.atlassian.net/browse/TOIL-933)
┆Issue Number: TOIL-933



Discussion:

Resolving patch:
diff --git a/contrib/admin/mypy-with-ignore.py b/contrib/admin/mypy-with-ignore.py
--- a/contrib/admin/mypy-with-ignore.py
+++ b/contrib/admin/mypy-with-ignore.py
@@ -32,7 +32,6 @@ def main():
         'src/toil/common.py',
         'src/toil/realtimeLogger.py',
         'src/toil/worker.py',
-        'src/toil/serviceManager.py',
         'src/toil/toilState.py',
         'src/toil/__init__.py',
         'src/toil/resource.py',
diff --git a/src/toil/serviceManager.py b/src/toil/serviceManager.py
--- a/src/toil/serviceManager.py
+++ b/src/toil/serviceManager.py
@@ -17,9 +17,12 @@
 import time
 from queue import Empty, Queue
 from threading import Event, Thread
+from typing import Dict, Set, Optional
 
 from toil.job import ServiceJobDescription
 from toil.lib.throttle import LocalThrottle, throttle
+from toil.jobStores.abstractJobStore import AbstractJobStore
+from toil.toilState import ToilState
 
 logger = logging.getLogger( __name__ )
 
@@ -27,27 +30,27 @@ class ServiceManager( object ):
     """
     Manages the scheduling of services.
     """
-    def __init__(self, jobStore, toilState):
+    def __init__(self, jobStore: AbstractJobStore, toilState: ToilState):
         logger.debug("Initializing service manager")
         self.jobStore = jobStore
 
         self.toilState = toilState
 
-        self.jobDescriptionsWithServicesBeingStarted = set()
+        self.jobDescriptionsWithServicesBeingStarted: Set[ServiceJobDescription] = set()
 
         self._terminate = Event() # This is used to terminate the thread associated
         # with the service manager
 
-        self._jobDescriptionsWithServicesToStart = Queue() # This is the input queue of
+        self._jobDescriptionsWithServicesToStart: Queue[ServiceJobDescription] = Queue() # This is the input queue of
         # JobDescriptions that have services that need to be started
 
-        self._jobDescriptionsWithServicesThatHaveStarted = Queue() # This is the output queue
+        self._jobDescriptionsWithServicesThatHaveStarted: Queue[ServiceJobDescription] = Queue() # This is the output queue
         # of JobDescriptions that have services that are already started
 
-        self._jobDescriptionsWithServicesThatHaveFailedToStart = Queue() # This is the output queue
+        self._jobDescriptionsWithServicesThatHaveFailedToStart: Queue[ServiceJobDescription] = Queue() # This is the output queue
         # of JobDescriptions that have services that are unable to start
 
-        self.serviceJobDescriptionsToStart = Queue() # This is the queue of services for the
+        self.serviceJobDescriptionsToStart: Queue[ServiceJobDescription] = Queue() # This is the queue of services for the
         # batch system to start
 
         self.jobsIssuedToServiceManager = 0 # The number of jobs the service manager
@@ -69,13 +72,13 @@ def __init__(self, jobStore, toilState):
 
 
 
-    def start(self):
+    def start(self) -> None:
         """
         Start the service scheduling thread.
         """
         self._serviceStarter.start()
 
-    def scheduleServices(self, jobDesc):
+    def scheduleServices(self, jobDesc: ServiceJobDescription) -> None:
         """
         Schedule the services of a job asynchronously.
         When the job's services are running the JobDescription for the job will
@@ -92,7 +95,7 @@ def scheduleServices(self, jobDesc):
         # Asynchronously schedule the services
         self._jobDescriptionsWithServicesToStart.put(jobDesc)
 
-    def getJobDescriptionWhoseServicesAreRunning(self, maxWait):
+    def getJobDescriptionWhoseServicesAreRunning(self, maxWait: float) -> Optional[ServiceJobDescription]:
         """
         :param float maxWait: Time in seconds to wait to get a JobDescription before returning
         :return: a JobDescription added to scheduleServices whose services are running, or None if
@@ -108,7 +111,7 @@ def getJobDescriptionWhoseServicesAreRunning(self, maxWait):
         except Empty:
             return None
 
-    def getJobDescriptionWhoseServicesFailedToStart(self, maxWait):
+    def getJobDescriptionWhoseServicesFailedToStart(self, maxWait: float) -> Optional[ServiceJobDescription]:
         """
         :param float maxWait: Time in seconds to wait to get a JobDescription before returning
         :return: a JobDescription added to scheduleServices whose services failed to start, or None if
@@ -124,7 +127,7 @@ def getJobDescriptionWhoseServicesFailedToStart(self, maxWait):
         except Empty:
             return None
 
-    def getServiceJobsToStart(self, maxWait):
+    def getServiceJobsToStart(self, maxWait: float) -> ServiceJobDescription:
         """
         :param float maxWait: Time in seconds to wait to get a job before returning.
         :return: a tuple of (serviceJobStoreID, memory, cores, disk, ..) representing
@@ -140,7 +143,7 @@ def getServiceJobsToStart(self, maxWait):
         except Empty:
             return None
 
-    def killServices(self, services, error=False):
+    def killServices(self, services: Dict[str, ServiceJobDescription], error: bool =False) -> None:
         """
         :param dict services: Maps service jobStoreIDs to the communication flags for the service
         """
@@ -150,7 +153,7 @@ def killServices(self, services, error=False):
                 self.jobStore.deleteFile(serviceJob.errorJobStoreID)
             self.jobStore.deleteFile(serviceJob.terminateJobStoreID)
 
-    def isActive(self, service):
+    def isActive(self, service: ServiceJobDescription) -> bool:
         """
         Returns true if the service job has not been told to terminate.
 
@@ -159,7 +162,7 @@ def isActive(self, service):
         """
         return self.jobStore.fileExists(service.terminateJobStoreID)
 
-    def isRunning(self, service):
+    def isRunning(self, service: ServiceJobDescription) -> bool:
         """
         Returns true if the service job has started and is active
 
@@ -168,7 +171,7 @@ def isRunning(self, service):
         """
         return (not self.jobStore.fileExists(service.startJobStoreID)) and self.isActive(service)
 
-    def check(self):
+    def check(self) -> None:
         """
         Check on the service manager thread.
         :raise RuntimeError: If the underlying thread has quit.
@@ -176,7 +179,7 @@ def check(self):
         if not self._serviceStarter.is_alive():
             raise RuntimeError("Service manager has quit")
 
-    def shutdown(self):
+    def shutdown(self) -> None:
         """
         Cleanly terminate worker threads starting and killing services. Will block
         until all services are started and blocked.
@@ -191,11 +194,12 @@ def shutdown(self):
         logger.debug('... finished shutting down the service manager. Took %s seconds', time.time() - startTime)
 
     @staticmethod
-    def _startServices(jobDescriptionsWithServicesToStart,
-                       jobDescriptionsWithServicesThatHaveStarted,
-                       jobDescriptionsWithServicesThatHaveFailedToStart,
-                       serviceJobsToStart,
-                       terminate, jobStore):
+    def _startServices(jobDescriptionsWithServicesToStart: 'Queue[ServiceJobDescription]',
+                       jobDescriptionsWithServicesThatHaveStarted: 'Queue[ServiceJobDescription]',
+                       jobDescriptionsWithServicesThatHaveFailedToStart: 'Queue[ServiceJobDescription]',
+                       serviceJobsToStart: 'Queue[ServiceJobDescription]',
+                       terminate: Event,
+                       jobStore: AbstractJobStore) -> None:
         """
         Thread used to schedule services.
         """
@@ -221,7 +225,6 @@ def _startServices(jobDescriptionsWithServicesToStart,
                         # ensure entire service "groups" are issued as a whole.
                         blockUntilServiceGroupIsStarted(jobDesc,
                                                         jobDescriptionsWithServicesThatHaveStarted,
-                                                        jobDescriptionsWithServicesThatHaveFailedToStart,
                                                         serviceJobsToStart, terminate, jobStore)
                         continue
                     # Found a new job that needs to schedule its services.
@@ -276,7 +279,11 @@ def _startServices(jobDescriptionsWithServicesToStart,
                 for jobDesc in jobDescriptionsToRemove:
                     del servicesRemainingToStartForJob[jobDesc]
 
-def blockUntilServiceGroupIsStarted(jobDesc, jobDescriptionsWithServicesThatHaveStarted, jobDescriptionsWithServicesThatHaveFailedToStart, serviceJobsToStart, terminate, jobStore):
+def blockUntilServiceGroupIsStarted(jobDesc: ServiceJobDescription,
+                                    jobDescriptionsWithServicesThatHaveStarted: 'Queue[ServiceJobDescription]',
+                                    serviceJobsToStart: 'Queue[ServiceJobDescription]',
+                                    terminate: Event,
+                                    jobStore: AbstractJobStore) -> None:
 
     # Keep the user informed, but not too informed, as services start up
     logLimiter = LocalThrottle(60)


Test patch: